diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 2a887e854a..26a80ba264 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -24,6 +24,7 @@ trigger: - requirements-doc.txt - doc/ - .ci/pipeline/docs.yml + - .circleci/config.yml pr: branches: @@ -35,6 +36,7 @@ pr: - requirements-doc.txt - doc/ - .ci/pipeline/docs.yml + - .circleci/config.yml variables: - name: MACOSX_DEPLOYMENT_TARGET diff --git a/.ci/scripts/describe_system.sh b/.ci/scripts/describe_system.sh index 6e16e67a99..038a0f7a1b 100755 --- a/.ci/scripts/describe_system.sh +++ b/.ci/scripts/describe_system.sh @@ -35,6 +35,10 @@ else echo "Unable to get operating system via uname or python/platform" fi echo +# meminfo +if [ -f /proc/meminfo ]; then + cat /proc/meminfo +fi # Compilers echo "Compilers:" if [ -x "$(command -v gcc)" ]; then diff --git a/.circleci/config.yml b/.circleci/config.yml index 3731ba0d27..1b462afb89 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,10 @@ jobs: name: Setting up build environment command: | ls -la + sudo apt-get update + sudo apt-get install -q -y gcc-10 g++-10 source ./.circleci/setup_env.sh + bash .ci/scripts/describe_system.sh - run: name: Building daal4py command: | @@ -37,7 +40,7 @@ jobs: conda activate bld export DALROOT=$CONDA_PREFIX export NO_DIST=1 - python setup.py install --single-version-externally-managed --record=record.txt + CC=gcc-10 CXX=g++-10 python setup.py install --single-version-externally-managed --record=record.txt - run: name: Building sklearnex command: | @@ -70,7 +73,10 @@ jobs: name: Setting up build environment command: | ls -la + sudo apt-get update + sudo apt-get install -q -y gcc-10 g++-10 source ./.circleci/setup_env.sh 3.10 + bash .ci/scripts/describe_system.sh - run: name: Building daal4py command: | @@ -78,7 +84,7 @@ jobs: conda activate bld export DALROOT=$CONDA_PREFIX export NO_DIST=1 - python setup.py install --single-version-externally-managed --record=record.txt + CC=gcc-10 CXX=g++-10 python setup.py install --single-version-externally-managed --record=record.txt - run: name: Testing sklearn patches no_output_timeout: 20m diff --git a/scripts/build_backend.py b/scripts/build_backend.py index 1951a1cd30..024c6a6a8c 100755 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -22,6 +22,8 @@ import subprocess from distutils import log from distutils.sysconfig import get_python_inc, get_config_var +import multiprocessing +from math import floor IS_WIN = False IS_MAC = False @@ -177,8 +179,16 @@ def custom_build_cmake_clib(iface, cxx=None, onedal_major_binary_version=1, no_d "-DONEDAL_DIST_SPMD:BOOL=ON" ] - import multiprocessing cpu_count = multiprocessing.cpu_count() + # limit parallel cmake jobs if memory size is insufficient + # TODO: add on all platforms + if IS_LIN: + with open('/proc/meminfo', 'r') as meminfo_file_obj: + memfree = meminfo_file_obj.read().split('\n')[1].split(' ') + while '' in memfree: + memfree.remove('') + memfree = int(memfree[1]) # total memory in kB + cpu_count = min(cpu_count, floor(max(1, memfree / 2 ** 20))) make_args = [ "cmake",