[Important] Use the MPI version of MUMPS

cpmech · Apr 29, 2024 · 38a7ecd · 38a7ecd
1 parent c877352
commit 38a7ecd
Show file tree

Hide file tree

Showing 15 changed files with 104 additions and 45 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -70,6 +70,7 @@
     "ldvt",
     "lgamma",
     "libmetis",
+    "libscalapack",
     "linalg",
     "lredrhs",
     "lrhs",
@@ -110,6 +111,7 @@
     "rwork",
     "Schur",
     "Schuster",
+    "setvars",
     "SLATEC",
     "Stegun",
     "substeps",
@@ -144,9 +146,5 @@
     "zscal",
     "zsyrk",
     "zticks"
-  ],
-  "rust-analyzer.linkedProjects": [
-    "./russell_lab/Cargo.toml"
-  ],
-  "rust-analyzer.showUnlinkedFileNotification": false
+  ]
 }
diff --git a/Dockerfile b/Dockerfile
@@ -19,7 +19,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \
   gdb \
   gfortran \
   liblapacke-dev \
-  libmumps-seq-dev \
+  libmumps-dev \
   libopenblas-dev \
   libsuitesparse-dev \
   && apt-get clean && rm -rf /var/lib/apt/lists/*

diff --git a/README.md b/README.md
@@ -78,6 +78,12 @@ russell_stat = { version = "*", features = ["intel_mkl"] }
 russell_tensor = { version = "*", features = ["intel_mkl"] }
 ```
 
+**Note:** To use the `intel_mkl` feature, the following command must be executed first:
+
+```bash
+source /opt/intel/oneapi/setvars.sh
+```
+
 External associated and recommended crates:
 
 - [plotpy](https://github.com/cpmech/plotpy) Plotting tools using Python3/Matplotlib as an engine (for quality graphics)
@@ -100,7 +106,7 @@ sudo apt-get install -y --no-install-recommends \
     gdb \
     gfortran \
     liblapacke-dev \
-    libmumps-seq-dev \
+    libmumps-dev \
     libopenblas-dev \
     libsuitesparse-dev
 ```
@@ -155,6 +161,12 @@ Run:
 bash case-b-intel-mkl-local-libs.bash
 ```
 
+**Note:** To use the `intel_mkl` feature, the following command must be executed first:
+
+```bash
+source /opt/intel/oneapi/setvars.sh
+```
+
 Then, add `intel_mkl` to your Cargo.toml or use `cargo build --features intel_mkl` (note that the `local_libs` feature will be automatically enabled).
 
 If locally compiled, the above scripts will save the resulting files in `/usr/local/lib/{mumps,umfpack}` and `/usr/local/include/{mumps,umfpack}`.

diff --git a/case-a-openblas-debian.bash b/case-a-openblas-debian.bash
@@ -15,6 +15,6 @@ sudo apt-get install -y --no-install-recommends \
     gdb \
     gfortran \
     liblapacke-dev \
-    libmumps-seq-dev \
+    libmumps-dev \
     libopenblas-dev \
     libsuitesparse-dev
diff --git a/case-a-openblas-local-libs.bash b/case-a-openblas-local-libs.bash
@@ -20,6 +20,8 @@ sudo apt-get install -y --no-install-recommends \
     liblapacke-dev \
     libmetis-dev \
     libopenblas-dev \
+    libopenmpi-dev \
+    libscalapack-mpi-dev \
     make \
     patch
 

diff --git a/russell_lab/README.md b/russell_lab/README.md
@@ -85,7 +85,7 @@ sudo apt-get install -y --no-install-recommends \
     gdb \
     gfortran \
     liblapacke-dev \
-    libmumps-seq-dev \
+    libmumps-dev \
     libopenblas-dev \
     libsuitesparse-dev
 ```

diff --git a/russell_ode/README.md b/russell_ode/README.md
@@ -77,7 +77,7 @@ sudo apt-get install -y --no-install-recommends \
     gdb \
     gfortran \
     liblapacke-dev \
-    libmumps-seq-dev \
+    libmumps-dev \
     libopenblas-dev \
     libsuitesparse-dev
 ```

diff --git a/russell_sparse/README.md b/russell_sparse/README.md
@@ -59,7 +59,7 @@ sudo apt-get install -y --no-install-recommends \
     gdb \
     gfortran \
     liblapacke-dev \
-    libmumps-seq-dev \
+    libmumps-dev \
     libopenblas-dev \
     libsuitesparse-dev
 ```

diff --git a/russell_sparse/build.rs b/russell_sparse/build.rs
@@ -1,12 +1,52 @@
+#[cfg(feature = "intel_mkl")]
+use std::env;
+
+#[cfg(feature = "intel_mkl")]
+fn validate_intel_setvars_completed() {
+    let intel_setvars_completed = match env::var("SETVARS_COMPLETED") {
+        Ok(v) => v == "1",
+        Err(_) => false,
+    };
+    if !intel_setvars_completed {
+        panic!("\n\nBUILD ERROR: Intel setvars.sh need to be sourced first.\nYou must execute the following command (just once):\nsource /opt/intel/oneapi/setvars.sh\n\n")
+    }
+}
+
+#[cfg(not(feature = "intel_mkl"))] // OpenBLAS + OpenMPI
+fn validate_intel_setvars_completed() {}
+
+#[cfg(feature = "intel_mkl")]
+fn get_mpi_include_dirs() -> Vec<String> {
+    vec!["/opt/intel/oneapi/mpi/latest/include/".to_string()]
+}
+
+#[cfg(not(feature = "intel_mkl"))] // OpenBLAS + OpenMPI
+fn get_mpi_include_dirs() -> Vec<String> {
+    vec!["/usr/lib/x86_64-linux-gnu/openmpi/include".to_string()]
+}
+
+#[cfg(feature = "intel_mkl")]
+fn get_mpi_link_dirs() -> Vec<String> {
+    vec!["/opt/intel/oneapi/mpi/latest/lib/".to_string()]
+}
+
+#[cfg(not(feature = "intel_mkl"))] // OpenBLAS + OpenMPI
+fn get_mpi_link_dirs() -> Vec<String> {
+    vec!["/usr/lib/x86_64-linux-gnu/openmpi".to_string()]
+}
+
 #[cfg(feature = "local_libs")]
 fn compile_libs() {
     // local MUMPS
     cc::Build::new()
         .file("c_code/interface_complex_mumps.c")
         .file("c_code/interface_mumps.c")
+        .includes(get_mpi_include_dirs())
         .include("/usr/local/include/mumps")
-        .include("/usr/lib/x86_64-linux-gnu/openmpi/include")
         .compile("c_code_interface_mumps");
+    for d in get_mpi_link_dirs() {
+        println!("cargo:rustc-link-search=native={d}");
+    }
     println!("cargo:rustc-link-search=native=/usr/local/lib/mumps");
     println!("cargo:rustc-link-lib=dylib=dmumps_cpmech");
     println!("cargo:rustc-link-lib=dylib=zmumps_cpmech");
@@ -26,15 +66,18 @@ fn compile_libs() {
     println!("cargo:rustc-cfg=local_umfpack");
 }
 
-#[cfg(not(feature = "local_libs"))]
+#[cfg(not(feature = "local_libs"))] // Libraries from the distribution
 fn compile_libs() {
     // MUMPS
     cc::Build::new()
         .file("c_code/interface_complex_mumps.c")
         .file("c_code/interface_mumps.c")
+        .includes(get_mpi_include_dirs())
         .include("/usr/include/mumps")
-        .include("/usr/lib/x86_64-linux-gnu/openmpi/include")
         .compile("c_code_interface_mumps");
+    for d in get_mpi_link_dirs() {
+        println!("cargo:rustc-link-search=native={d}");
+    }
     println!("cargo:rustc-link-lib=dylib=dmumps");
     println!("cargo:rustc-link-lib=dylib=zmumps");
     println!("cargo:rustc-link-lib=dylib=mpi");
@@ -51,5 +94,6 @@ fn compile_libs() {
 }
 
 fn main() {
+    validate_intel_setvars_completed();
     compile_libs();
 }
diff --git a/russell_stat/README.md b/russell_stat/README.md
@@ -57,7 +57,7 @@ sudo apt-get install -y --no-install-recommends \
     gdb \
     gfortran \
     liblapacke-dev \
-    libmumps-seq-dev \
+    libmumps-dev \
     libopenblas-dev \
     libsuitesparse-dev
 ```

diff --git a/russell_tensor/README.md b/russell_tensor/README.md
@@ -43,7 +43,7 @@ sudo apt-get install -y --no-install-recommends \
     gdb \
     gfortran \
     liblapacke-dev \
-    libmumps-seq-dev \
+    libmumps-dev \
     libopenblas-dev \
     libsuitesparse-dev
 ```

diff --git a/zscripts/install-intel-mkl-and-ifort-debian.bash b/zscripts/install-intel-mkl-and-ifort-debian.bash
@@ -19,7 +19,9 @@ sudo apt-get update -y && \
 sudo apt-get install -y --no-install-recommends \
     intel-oneapi-compiler-fortran-$VERSION \
     intel-oneapi-mkl-$VERSION \
-    intel-oneapi-mkl-devel-$VERSION
+    intel-oneapi-mkl-devel-$VERSION \
+    intel-oneapi-mpi \
+    intel-oneapi-mpi-devel
 
 LIBDIR1="/opt/intel/oneapi/mkl/$VERSION/lib/intel64"
 LIBDIR2="/opt/intel/oneapi/compiler/$VERSION/linux/compiler/lib/intel64_lin"

diff --git a/zscripts/makefiles-mumps/Makefile.inc b/zscripts/makefiles-mumps/Makefile.inc
@@ -1,12 +1,7 @@
 #
-#  This file is part of MUMPS 5.6.1, released
-#  on Tue Jul 11 07:51:28 UTC 2023
+#  This file is part of MUMPS 5.6.2, released
+#  on Wed Oct 11 09:36:25 UTC 2023
 #
-# These settings for a PC under Debian/linux with standard packages :
-# metis (parmetis), scotch (ptscotch), openmpi, gfortran
-
-# packages installation: 
-# apt-get install gcc gfortran libmetis-dev libopenblas-dev
 
 # must be at the top
 PLAT = _cpmech
@@ -22,7 +17,7 @@ IPORD    = -I$(topdir)/PORD/include/
 LPORD    = -L$(LPORDDIR) -lpord$(PLAT)
 
 LMETISDIR = /usr/lib 
-IMETIS    = -I/usr/include/metis
+IMETIS    = -I/usr/include
 
 LMETIS    = -L$(LMETISDIR) -lmetis
 
@@ -46,14 +41,17 @@ LIBEXT  = .a
 OUTC    = -o 
 OUTF    = -o 
 RM = /bin/rm -f
-CC = gcc
-FC = gfortran
-FL = gfortran
+CC = mpicc
+FC = mpif90
+FL = mpif90
 AR = ar vr 
 RANLIB = ranlib
 LAPACK = -llapack
+SCALAP  = -lscalapack-openmpi # -lblacs-openmpi
 
+INCPAR = # not needed with mpif90/mpicc:  -I/usr/include/openmpi
 
+LIBPAR = $(SCALAP) $(LAPACK) # not needed with mpif90/mpicc: -lmpi_mpifh -lmpi
 
 INCSEQ = -I$(topdir)/libseq
 LIBSEQ  = $(LAPACK) -L$(topdir)/libseq -lmpiseq$(PLAT)
@@ -71,7 +69,8 @@ OPTF    = -O -fopenmp -fallow-argument-mismatch
 OPTL    = -O -fopenmp
 OPTC    = -O -fopenmp
 #End Optimized options
-
-INCS = $(INCSEQ)
-LIBS = $(LIBSEQ)
-LIBSEQNEEDED = libseqneeded
+
+INCS = $(INCPAR)
+LIBS = $(LIBPAR)
+LIBSEQNEEDED =
+
diff --git a/zscripts/makefiles-mumps/MakefileMKL.inc b/zscripts/makefiles-mumps/MakefileMKL.inc
@@ -1,9 +1,7 @@
 #
-#  This file is part of MUMPS 5.6.1, released
-#  on Tue Jul 11 07:51:28 UTC 2023
+#  This file is part of MUMPS 5.6.2, released
+#  on Wed Oct 11 09:36:25 UTC 2023
 #
-# These settings for a PC under Debian/linux with standard packages :
-# metis (parmetis), scotch (ptscotch), openmpi 
 
 # must be at the top
 PLAT = _cpmech
@@ -19,7 +17,7 @@ IPORD    = -I$(topdir)/PORD/include/
 LPORD    = -L$(LPORDDIR) -lpord$(PLAT)
 
 LMETISDIR = /usr/lib 
-IMETIS    = -I/usr/include/metis
+IMETIS    = -I/usr/include
 
 LMETIS    = -L$(LMETISDIR) -lmetis
 
@@ -43,15 +41,18 @@ LIBEXT  = .a
 OUTC    = -o 
 OUTF    = -o 
 RM = /bin/rm -f
-CC = gcc
-FC = ifort
-FL = ifort
+CC = mpicc
+FC = mpiifort
+FL = mpiifort
 AR = ar vr 
 RANLIB = ranlib
 MKLROOT=/opt/intel/oneapi/mkl/latest/lib/intel64
 LAPACK = -L$(MKLROOT) -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core
+SCALAP = -L$(MKLROOT) -lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64
 
+INCPAR = # not needed with mpif90/mpicc:  -I/usr/include/openmpi
 
+LIBPAR = $(SCALAP) $(LAPACK)
 
 INCSEQ = -I$(topdir)/libseq
 LIBSEQ  = $(LAPACK) -L$(topdir)/libseq -lmpiseq$(PLAT)
@@ -68,7 +69,8 @@ OPTF    = -O -nofor-main -fopenmp -DGEMMT_AVAILABLE  -diag-disable=10448
 OPTL    = -O -nofor-main -fopenmp
 OPTC    = -O -fopenmp
 #End Optimized options
-
-INCS = $(INCSEQ)
-LIBS = $(LIBSEQ)
-LIBSEQNEEDED = libseqneeded
+
+INCS = $(INCPAR)
+LIBS = $(LIBPAR)
+LIBSEQNEEDED =
+
diff --git a/zscripts/ubuntu-remove-deps.bash b/zscripts/ubuntu-remove-deps.bash
@@ -5,7 +5,7 @@ set -e
 sudo apt-get remove \
     gfortran \
     liblapacke-dev \
-    libmumps-seq-dev \
+    libmumps-dev \
     libopenblas-dev \
     libsuitesparse-dev