Update gcc version in CircleCI config; limit cmake jobs (#1196)

* Update GCC version used in circleci * Exclude circleci config from azure pipelines trigger * Limit number of cmake jobs * Add meminfo to 'describe_system' script
intel · Feb 27, 2023 · 94744b2 · 94744b2
1 parent 9928722
commit 94744b2
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 3 deletions.
diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml
@@ -24,6 +24,7 @@ trigger:
     - requirements-doc.txt
     - doc/
     - .ci/pipeline/docs.yml
+    - .circleci/config.yml
 
 pr:
   branches:
@@ -35,6 +36,7 @@ pr:
     - requirements-doc.txt
     - doc/
     - .ci/pipeline/docs.yml
+    - .circleci/config.yml
 
 variables:
   - name: MACOSX_DEPLOYMENT_TARGET

diff --git a/.ci/scripts/describe_system.sh b/.ci/scripts/describe_system.sh
@@ -35,6 +35,10 @@ else
     echo "Unable to get operating system via uname or python/platform"
 fi
 echo
+# meminfo
+if [ -f /proc/meminfo ]; then
+    cat /proc/meminfo
+fi
 # Compilers
 echo "Compilers:"
 if [ -x "$(command -v gcc)" ]; then

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -29,15 +29,18 @@ jobs:
           name: Setting up build environment
           command: |
             ls -la
+            sudo apt-get update
+            sudo apt-get install -q -y gcc-10 g++-10
             source ./.circleci/setup_env.sh
+            bash .ci/scripts/describe_system.sh
       - run:
           name: Building daal4py
           command: |
             . ~/miniconda/etc/profile.d/conda.sh
             conda activate bld
             export DALROOT=$CONDA_PREFIX
             export NO_DIST=1
-            python setup.py install --single-version-externally-managed --record=record.txt
+            CC=gcc-10 CXX=g++-10 python setup.py install --single-version-externally-managed --record=record.txt
       - run:
           name: Building sklearnex
           command: |
@@ -70,15 +73,18 @@ jobs:
           name: Setting up build environment
           command: |
             ls -la
+            sudo apt-get update
+            sudo apt-get install -q -y gcc-10 g++-10
             source ./.circleci/setup_env.sh 3.10
+            bash .ci/scripts/describe_system.sh
       - run:
           name: Building daal4py
           command: |
             . ~/miniconda/etc/profile.d/conda.sh
             conda activate bld
             export DALROOT=$CONDA_PREFIX
             export NO_DIST=1
-            python setup.py install --single-version-externally-managed --record=record.txt
+            CC=gcc-10 CXX=g++-10 python setup.py install --single-version-externally-managed --record=record.txt
       - run:
           name: Testing sklearn patches
           no_output_timeout: 20m

diff --git a/scripts/build_backend.py b/scripts/build_backend.py
@@ -22,6 +22,8 @@
 import subprocess
 from distutils import log
 from distutils.sysconfig import get_python_inc, get_config_var
+import multiprocessing
+from math import floor
 
 IS_WIN = False
 IS_MAC = False
@@ -177,8 +179,16 @@ def custom_build_cmake_clib(iface, cxx=None, onedal_major_binary_version=1, no_d
             "-DONEDAL_DIST_SPMD:BOOL=ON"
         ]
 
-    import multiprocessing
     cpu_count = multiprocessing.cpu_count()
+    # limit parallel cmake jobs if memory size is insufficient
+    # TODO: add on all platforms
+    if IS_LIN:
+        with open('/proc/meminfo', 'r') as meminfo_file_obj:
+            memfree = meminfo_file_obj.read().split('\n')[1].split(' ')
+            while '' in memfree:
+                memfree.remove('')
+            memfree = int(memfree[1])  # total memory in kB
+        cpu_count = min(cpu_count, floor(max(1, memfree / 2 ** 20)))
 
     make_args = [
         "cmake",