From 3588a45e47c89a94a52963cdda39fb5fb479a4fb Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 11 Jul 2023 11:42:14 -0700 Subject: [PATCH 1/9] feat: add black GHA for changed files --- .github/workflows/black.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/black.yml diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml new file mode 100644 index 0000000000..606f29a91e --- /dev/null +++ b/.github/workflows/black.yml @@ -0,0 +1,25 @@ +name: Lint + +on: [pull_request, push] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - id: changedfiles + name: Get list of changed .py files + run: | + echo "CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep .py$ | xargs)" >> "$GITHUB_OUTPUT" + if [[ $(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep .py$ | wc -l) -eq 0 ]]; then + echo "HAS_CHANGES=false" >> "$GITHUB_OUTPUT" + else + echo "HAS_CHANGES=true" >> "$GITHUB_OUTPUT" + fi + - uses: psf/black@stable + name: Run black on changed files + if: steps.changedfiles.outputs.HAS_CHANGES == 'true' + with: + src: ${{ steps.changedfiles.outputs.CHANGED_FILES }} From a144a9fb4c8f8cc1b5ca6c3491e421c9fd40c4a4 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 12 Jul 2023 02:46:48 -0700 Subject: [PATCH 2/9] test: small change to trigger GHA --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8ee45bb2a9..2358395b7e 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ try: import dpctl - dpctl_available = dpctl.__version__ >= '0.14' + dpctl_available = dpctl.__version__ >= "0.14" except ImportError: dpctl_available = False From 4f88e5f77f7b0109b2d9316716cccae87af8cce5 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 12 Jul 2023 04:16:50 -0700 Subject: [PATCH 3/9] black format / test pre-commit hook --- setup.py | 469 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 257 insertions(+), 212 deletions(-) diff --git a/setup.py b/setup.py index 2358395b7e..6f0a24ee9c 100644 --- a/setup.py +++ b/setup.py @@ -43,43 +43,49 @@ IS_MAC = False IS_LIN = False -dal_root = os.environ.get('DALROOT') +dal_root = os.environ.get("DALROOT") if dal_root is None: raise RuntimeError("Not set DALROOT variable") -if 'linux' in sys.platform: +if "linux" in sys.platform: IS_LIN = True - lib_dir = jp(dal_root, 'lib', 'intel64') -elif sys.platform == 'darwin': + lib_dir = jp(dal_root, "lib", "intel64") +elif sys.platform == "darwin": IS_MAC = True - lib_dir = jp(dal_root, 'lib') -elif sys.platform in ['win32', 'cygwin']: + lib_dir = jp(dal_root, "lib") +elif sys.platform in ["win32", "cygwin"]: IS_WIN = True - lib_dir = jp(dal_root, 'lib', 'intel64') + lib_dir = jp(dal_root, "lib", "intel64") else: - assert False, sys.platform + ' not supported' + assert False, sys.platform + " not supported" ONEDAL_MAJOR_BINARY_VERSION, ONEDAL_MINOR_BINARY_VERSION = get_onedal_version( - dal_root, 'binary') + dal_root, "binary" +) ONEDAL_VERSION = get_onedal_version(dal_root) ONEDAL_2021_3 = 2021 * 10000 + 3 * 100 ONEDAL_2023_0_1 = 2023 * 10000 + 0 * 100 + 1 -is_onedal_iface = os.environ.get( - 'OFF_ONEDAL_IFACE') is None and ONEDAL_VERSION >= ONEDAL_2021_3 +is_onedal_iface = ( + os.environ.get("OFF_ONEDAL_IFACE") is None and ONEDAL_VERSION >= ONEDAL_2021_3 +) -d4p_version = (os.environ['DAAL4PY_VERSION'] if 'DAAL4PY_VERSION' in os.environ - else time.strftime('%Y%m%d.%H%M%S')) +d4p_version = ( + os.environ["DAAL4PY_VERSION"] + if "DAAL4PY_VERSION" in os.environ + else time.strftime("%Y%m%d.%H%M%S") +) -trues = ['true', 'True', 'TRUE', '1', 't', 'T', 'y', 'Y', 'Yes', 'yes', 'YES'] -no_dist = True if 'NO_DIST' in os.environ and os.environ['NO_DIST'] in trues else False -no_stream = 'NO_STREAM' in os.environ and os.environ['NO_STREAM'] in trues -mpi_root = None if no_dist else os.environ['MPIROOT'] -dpcpp = True if 'DPCPPROOT' in os.environ else False -dpcpp_root = None if not dpcpp else os.environ['DPCPPROOT'] +trues = ["true", "True", "TRUE", "1", "t", "T", "y", "Y", "Yes", "yes", "YES"] +no_dist = True if "NO_DIST" in os.environ and os.environ["NO_DIST"] in trues else False +no_stream = "NO_STREAM" in os.environ and os.environ["NO_STREAM"] in trues +mpi_root = None if no_dist else os.environ["MPIROOT"] +dpcpp = True if "DPCPPROOT" in os.environ else False +dpcpp_root = None if not dpcpp else os.environ["DPCPPROOT"] try: import dpctl + dpctl_available = dpctl.__version__ >= "0.14" except ImportError: dpctl_available = False @@ -87,16 +93,15 @@ build_distribute = dpcpp and dpctl_available and not no_dist and IS_LIN -daal_lib_dir = lib_dir if (IS_MAC or os.path.isdir( - lib_dir)) else os.path.dirname(lib_dir) +daal_lib_dir = lib_dir if (IS_MAC or os.path.isdir(lib_dir)) else os.path.dirname(lib_dir) ONEDAL_LIBDIRS = [daal_lib_dir] if IS_WIN: ONEDAL_LIBDIRS.append(f"{os.environ.get('CONDA_PREFIX')}/Library/lib") if no_stream: - print('\nDisabling support for streaming mode\n') + print("\nDisabling support for streaming mode\n") if no_dist: - print('\nDisabling support for distributed mode\n') + print("\nDisabling support for distributed mode\n") DIST_CFLAGS = [] DIST_CPPS = [] MPI_INCDIRS = [] @@ -104,113 +109,147 @@ MPI_LIBS = [] MPI_CPPS = [] else: - DIST_CFLAGS = ['-D_DIST_', ] - DIST_CPPS = ['src/transceiver.cpp'] - MPI_INCDIRS = [jp(mpi_root, 'include')] - MPI_LIBDIRS = [jp(mpi_root, 'lib')] - MPI_LIBNAME = getattr(os.environ, 'MPI_LIBNAME', None) + DIST_CFLAGS = [ + "-D_DIST_", + ] + DIST_CPPS = ["src/transceiver.cpp"] + MPI_INCDIRS = [jp(mpi_root, "include")] + MPI_LIBDIRS = [jp(mpi_root, "lib")] + MPI_LIBNAME = getattr(os.environ, "MPI_LIBNAME", None) if MPI_LIBNAME: MPI_LIBS = [MPI_LIBNAME] elif IS_WIN: - if os.path.isfile(jp(mpi_root, 'lib', 'mpi.lib')): - MPI_LIBS = ['mpi'] - if os.path.isfile(jp(mpi_root, 'lib', 'impi.lib')): - MPI_LIBS = ['impi'] + if os.path.isfile(jp(mpi_root, "lib", "mpi.lib")): + MPI_LIBS = ["mpi"] + if os.path.isfile(jp(mpi_root, "lib", "impi.lib")): + MPI_LIBS = ["impi"] assert MPI_LIBS, "Couldn't find MPI library" else: - MPI_LIBS = ['mpi'] - MPI_CPPS = ['src/mpi/mpi_transceiver.cpp'] + MPI_LIBS = ["mpi"] + MPI_CPPS = ["src/mpi/mpi_transceiver.cpp"] def get_sdl_cflags(): if IS_LIN or IS_MAC: - return DIST_CFLAGS + ['-fstack-protector-strong', '-fPIC', - '-D_FORTIFY_SOURCE=2', '-Wformat', - '-Wformat-security', '-fno-strict-overflow', - '-fno-delete-null-pointer-checks'] + return DIST_CFLAGS + [ + "-fstack-protector-strong", + "-fPIC", + "-D_FORTIFY_SOURCE=2", + "-Wformat", + "-Wformat-security", + "-fno-strict-overflow", + "-fno-delete-null-pointer-checks", + ] if IS_WIN: - return DIST_CFLAGS + ['-GS'] + return DIST_CFLAGS + ["-GS"] def get_sdl_ldflags(): if IS_LIN: - return ['-Wl,-z,noexecstack,-z,relro,-z,now,-fstack-protector-strong,' - '-fno-strict-overflow,-fno-delete-null-pointer-checks,-fwrapv'] + return [ + "-Wl,-z,noexecstack,-z,relro,-z,now,-fstack-protector-strong," + "-fno-strict-overflow,-fno-delete-null-pointer-checks,-fwrapv" + ] if IS_MAC: - return ['-fstack-protector-strong', - '-fno-strict-overflow', - '-fno-delete-null-pointer-checks', - '-fwrapv'] + return [ + "-fstack-protector-strong", + "-fno-strict-overflow", + "-fno-delete-null-pointer-checks", + "-fwrapv", + ] if IS_WIN: - return ['-NXCompat', '-DynamicBase'] + return ["-NXCompat", "-DynamicBase"] def get_daal_type_defines(): - daal_type_defines = ['DAAL_ALGORITHM_FP_TYPE', - 'DAAL_SUMMARY_STATISTICS_TYPE', - 'DAAL_DATA_TYPE'] - return [(d, 'double') for d in daal_type_defines] + daal_type_defines = [ + "DAAL_ALGORITHM_FP_TYPE", + "DAAL_SUMMARY_STATISTICS_TYPE", + "DAAL_DATA_TYPE", + ] + return [(d, "double") for d in daal_type_defines] -def get_libs(iface='daal'): +def get_libs(iface="daal"): major_version = ONEDAL_MAJOR_BINARY_VERSION if IS_WIN: - libraries_plat = [f'onedal_core_dll.{major_version}'] - onedal_lib = [f'onedal_dll.{major_version}'] - onedal_dpc_lib = [f'onedal_dpc_dll.{major_version}'] + libraries_plat = [f"onedal_core_dll.{major_version}"] + onedal_lib = [f"onedal_dll.{major_version}"] + onedal_dpc_lib = [f"onedal_dpc_dll.{major_version}"] elif IS_MAC: - libraries_plat = [f'onedal_core.{major_version}', - f'onedal_thread.{major_version}'] - onedal_lib = [f'onedal.{major_version}'] - onedal_dpc_lib = [f'onedal_dpc.{major_version}'] + libraries_plat = [ + f"onedal_core.{major_version}", + f"onedal_thread.{major_version}", + ] + onedal_lib = [f"onedal.{major_version}"] + onedal_dpc_lib = [f"onedal_dpc.{major_version}"] else: - libraries_plat = [f':libonedal_core.so.{major_version}', - f':libonedal_thread.so.{major_version}'] - onedal_lib = [f':libonedal.so.{major_version}'] - onedal_dpc_lib = [f':libonedal_dpc.so.{major_version}'] - if iface == 'onedal': + libraries_plat = [ + f":libonedal_core.so.{major_version}", + f":libonedal_thread.so.{major_version}", + ] + onedal_lib = [f":libonedal.so.{major_version}"] + onedal_dpc_lib = [f":libonedal_dpc.so.{major_version}"] + if iface == "onedal": libraries_plat = onedal_lib + libraries_plat - elif iface == 'onedal_dpc': + elif iface == "onedal_dpc": libraries_plat = onedal_dpc_lib + libraries_plat return libraries_plat def get_build_options(): - include_dir_plat = [os.path.abspath('./src'), - os.path.abspath('.'), - dal_root + '/include', ] + include_dir_plat = [ + os.path.abspath("./src"), + os.path.abspath("."), + dal_root + "/include", + ] # FIXME it is a wrong place for this dependency if not no_dist: - include_dir_plat.append(mpi_root + '/include') - using_intel = os.environ.get('cc', '') in [ - 'icc', 'icpc', 'icl', 'dpcpp', 'icx', 'icpx'] - eca = ['-DPY_ARRAY_UNIQUE_SYMBOL=daal4py_array_API', - '-DD4P_VERSION="' + d4p_version + '"', '-DNPY_ALLOW_THREADS=1'] + include_dir_plat.append(mpi_root + "/include") + using_intel = os.environ.get("cc", "") in [ + "icc", + "icpc", + "icl", + "dpcpp", + "icx", + "icpx", + ] + eca = [ + "-DPY_ARRAY_UNIQUE_SYMBOL=daal4py_array_API", + '-DD4P_VERSION="' + d4p_version + '"', + "-DNPY_ALLOW_THREADS=1", + ] ela = [] if using_intel and IS_WIN: include_dir_plat.append( - jp(os.environ.get('ICPP_COMPILER16', ''), 'compiler', 'include')) - eca += ['-std=c++17', '-w', '/MD'] + jp(os.environ.get("ICPP_COMPILER16", ""), "compiler", "include") + ) + eca += ["-std=c++17", "-w", "/MD"] elif not using_intel and IS_WIN: - eca += ['-wd4267', '-wd4244', '-wd4101', '-wd4996', '/std:c++17'] + eca += ["-wd4267", "-wd4244", "-wd4101", "-wd4996", "/std:c++17"] else: - eca += ['-std=c++17', '-w', ] # '-D_GLIBCXX_USE_CXX11_ABI=0'] + eca += [ + "-std=c++17", + "-w", + ] # '-D_GLIBCXX_USE_CXX11_ABI=0'] # Security flags eca += get_sdl_cflags() ela += get_sdl_ldflags() if IS_MAC: - eca.append('-stdlib=libc++') - ela.append('-stdlib=libc++') + eca.append("-stdlib=libc++") + ela.append("-stdlib=libc++") ela.append("-Wl,-rpath,{}".format(daal_lib_dir)) ela.append("-Wl,-rpath,@loader_path/../../../") elif IS_WIN: - ela.append('-IGNORE:4197') - elif IS_LIN and not any(x in os.environ and '-g' in os.environ[x] - for x in ['CPPFLAGS', 'CFLAGS', 'LDFLAGS']): - ela.append('-s') + ela.append("-IGNORE:4197") + elif IS_LIN and not any( + x in os.environ and "-g" in os.environ[x] + for x in ["CPPFLAGS", "CFLAGS", "LDFLAGS"] + ): + ela.append("-s") if IS_LIN: ela.append("-fPIC") ela.append("-Wl,-rpath,$ORIGIN/../../../") @@ -223,18 +262,23 @@ def getpyexts(): exts = [] - ext = Extension('daal4py._daal4py', - [os.path.abspath('src/daal4py.cpp'), - os.path.abspath('build/daal4py_cpp.cpp'), - os.path.abspath('build/daal4py_cy.pyx')] + DIST_CPPS, - depends=glob.glob(jp(os.path.abspath('src'), '*.h')), - include_dirs=include_dir_plat + [np.get_include()], - extra_compile_args=eca, - define_macros=get_daal_type_defines(), - extra_link_args=ela, - libraries=libraries_plat, - library_dirs=ONEDAL_LIBDIRS, - language='c++') + ext = Extension( + "daal4py._daal4py", + [ + os.path.abspath("src/daal4py.cpp"), + os.path.abspath("build/daal4py_cpp.cpp"), + os.path.abspath("build/daal4py_cy.pyx"), + ] + + DIST_CPPS, + depends=glob.glob(jp(os.path.abspath("src"), "*.h")), + include_dirs=include_dir_plat + [np.get_include()], + extra_compile_args=eca, + define_macros=get_daal_type_defines(), + extra_link_args=ela, + libraries=libraries_plat, + library_dirs=ONEDAL_LIBDIRS, + language="c++", + ) exts.extend(cythonize(ext)) if dpcpp: @@ -243,105 +287,111 @@ def getpyexts(): elif IS_WIN: runtime_oneapi_dirs = [] - ext = Extension('daal4py._oneapi', - [os.path.abspath('src/oneapi/oneapi.pyx'), ], - depends=['src/oneapi/oneapi.h', 'src/oneapi/oneapi_backend.h'], - include_dirs=include_dir_plat + [np.get_include()], - extra_compile_args=eca, - extra_link_args=ela, - define_macros=[ - ('NPY_NO_DEPRECATED_API', - 'NPY_1_7_API_VERSION') - ], - libraries=['oneapi_backend'] + libraries_plat, - library_dirs=['daal4py/oneapi'] + ONEDAL_LIBDIRS, - runtime_library_dirs=runtime_oneapi_dirs, - language='c++') + ext = Extension( + "daal4py._oneapi", + [ + os.path.abspath("src/oneapi/oneapi.pyx"), + ], + depends=["src/oneapi/oneapi.h", "src/oneapi/oneapi_backend.h"], + include_dirs=include_dir_plat + [np.get_include()], + extra_compile_args=eca, + extra_link_args=ela, + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")], + libraries=["oneapi_backend"] + libraries_plat, + library_dirs=["daal4py/oneapi"] + ONEDAL_LIBDIRS, + runtime_library_dirs=runtime_oneapi_dirs, + language="c++", + ) exts.extend(cythonize(ext)) if not no_dist: mpi_include_dir = include_dir_plat + [np.get_include()] + MPI_INCDIRS - mpi_depens = glob.glob(jp(os.path.abspath('src'), '*.h')) + mpi_depens = glob.glob(jp(os.path.abspath("src"), "*.h")) mpi_extra_link = ela + ["-Wl,-rpath,{}".format(x) for x in MPI_LIBDIRS] - exts.append(Extension('daal4py.mpi_transceiver', - MPI_CPPS, - depends=mpi_depens, - include_dirs=mpi_include_dir, - extra_compile_args=eca, - define_macros=get_daal_type_defines(), - extra_link_args=mpi_extra_link, - libraries=libraries_plat + MPI_LIBS, - library_dirs=ONEDAL_LIBDIRS + MPI_LIBDIRS, - language='c++')) + exts.append( + Extension( + "daal4py.mpi_transceiver", + MPI_CPPS, + depends=mpi_depens, + include_dirs=mpi_include_dir, + extra_compile_args=eca, + define_macros=get_daal_type_defines(), + extra_link_args=mpi_extra_link, + libraries=libraries_plat + MPI_LIBS, + library_dirs=ONEDAL_LIBDIRS + MPI_LIBDIRS, + language="c++", + ) + ) return exts cfg_vars = get_config_vars() for key, value in get_config_vars().items(): if isinstance(value, str): - cfg_vars[key] = value.replace( - "-Wstrict-prototypes", "").replace('-DNDEBUG', '') + cfg_vars[key] = value.replace("-Wstrict-prototypes", "").replace("-DNDEBUG", "") def gen_pyx(odir): - gtr_files = glob.glob( - jp(os.path.abspath('generator'), '*')) + ['./setup.py'] - src_files = [os.path.abspath('build/daal4py_cpp.h'), - os.path.abspath('build/daal4py_cpp.cpp'), - os.path.abspath('build/daal4py_cy.pyx')] + gtr_files = glob.glob(jp(os.path.abspath("generator"), "*")) + ["./setup.py"] + src_files = [ + os.path.abspath("build/daal4py_cpp.h"), + os.path.abspath("build/daal4py_cpp.cpp"), + os.path.abspath("build/daal4py_cy.pyx"), + ] if all(os.path.isfile(x) for x in src_files): src_files.sort(key=os.path.getmtime) gtr_files.sort(key=os.path.getmtime, reverse=True) if os.path.getmtime(src_files[0]) > os.path.getmtime(gtr_files[0]): - print('Generated files are all newer than generator code.' - 'Skipping code generation') + print( + "Generated files are all newer than generator code." + "Skipping code generation" + ) return from generator.gen_daal4py import gen_daal4py + odir = os.path.abspath(odir) if not os.path.isdir(odir): os.mkdir(odir) - gen_daal4py(dal_root, odir, d4p_version, - no_dist=no_dist, no_stream=no_stream) + gen_daal4py(dal_root, odir, d4p_version, no_dist=no_dist, no_stream=no_stream) -gen_pyx(os.path.abspath('./build')) +gen_pyx(os.path.abspath("./build")) def build_oneapi_backend(): eca, ela, includes = get_build_options() - cc = 'icx' + cc = "icx" if IS_WIN: - cxx = 'icx' + cxx = "icx" else: - cxx = 'icpx' - eca = ['-fsycl'] + ['-fsycl-device-code-split=per_kernel'] + eca - ela = ['-fsycl'] + ['-fsycl-device-code-split=per_kernel'] + ela + cxx = "icpx" + eca = ["-fsycl"] + ["-fsycl-device-code-split=per_kernel"] + eca + ela = ["-fsycl"] + ["-fsycl-device-code-split=per_kernel"] + ela return build_backend.build_cpp( cc=cc, cxx=cxx, - sources=['src/oneapi/oneapi_backend.cpp'], - targetname='oneapi_backend', - targetprefix='' if IS_WIN else 'lib', - targetsuffix='.dll' if IS_WIN else '.so', - libs=get_libs('daal') + ['OpenCL', 'onedal_sycl'], + sources=["src/oneapi/oneapi_backend.cpp"], + targetname="oneapi_backend", + targetprefix="" if IS_WIN else "lib", + targetsuffix=".dll" if IS_WIN else ".so", + libs=get_libs("daal") + ["OpenCL", "onedal_sycl"], libdirs=ONEDAL_LIBDIRS, includes=includes, eca=eca, ela=ela, defines=[], - installpath='daal4py/oneapi/' + installpath="daal4py/oneapi/", ) def get_onedal_py_libs(): - ext_suffix = get_config_vars('EXT_SUFFIX')[0] - libs = [f'_onedal_py_host{ext_suffix}', f'_onedal_py_dpc{ext_suffix}'] + ext_suffix = get_config_vars("EXT_SUFFIX")[0] + libs = [f"_onedal_py_host{ext_suffix}", f"_onedal_py_dpc{ext_suffix}"] if IS_WIN: - ext_suffix_lib = ext_suffix.replace('.dll', '.lib') - libs += [f'_onedal_py_host{ext_suffix_lib}', - f'_onedal_py_dpc{ext_suffix_lib}'] + ext_suffix_lib = ext_suffix.replace(".dll", ".lib") + libs += [f"_onedal_py_host{ext_suffix_lib}", f"_onedal_py_dpc{ext_suffix_lib}"] return libs @@ -353,39 +403,41 @@ def build_extensions(self): executor.submit(self.build_extension, ext) -class custom_build(): +class custom_build: def run(self): if is_onedal_iface: - cxx = os.getenv('CXX', 'cl' if IS_WIN else 'g++') + cxx = os.getenv("CXX", "cl" if IS_WIN else "g++") build_backend.custom_build_cmake_clib( - 'host', cxx, ONEDAL_MAJOR_BINARY_VERSION, no_dist=no_dist) + "host", cxx, ONEDAL_MAJOR_BINARY_VERSION, no_dist=no_dist + ) if dpcpp: build_oneapi_backend() if is_onedal_iface: build_backend.custom_build_cmake_clib( - 'dpc', ONEDAL_MAJOR_BINARY_VERSION, no_dist=no_dist) + "dpc", ONEDAL_MAJOR_BINARY_VERSION, no_dist=no_dist + ) def post_build(self): if IS_MAC: import subprocess + # manually fix incorrect install_name of oneDAL 2023.0.1 libs major_version = ONEDAL_MAJOR_BINARY_VERSION - major_is_available = find_library( - f'libonedal_core.{major_version}.dylib') is not None + major_is_available = ( + find_library(f"libonedal_core.{major_version}.dylib") is not None + ) if major_is_available and ONEDAL_VERSION == ONEDAL_2023_0_1: - extension_libs = list(pathlib.Path('.').glob('**/*darwin.so')) - onedal_libs = [ - 'onedal', - 'onedal_dpc', - 'onedal_core', - 'onedal_thread'] + extension_libs = list(pathlib.Path(".").glob("**/*darwin.so")) + onedal_libs = ["onedal", "onedal_dpc", "onedal_core", "onedal_thread"] for ext_lib in extension_libs: for onedal_lib in onedal_libs: subprocess.call( - '/usr/bin/install_name_tool -change ' - f'lib{onedal_lib}.dylib ' - f'lib{onedal_lib}.{major_version}.dylib ' - f'{ext_lib}'.split(' '), shell=False) + "/usr/bin/install_name_tool -change " + f"lib{onedal_lib}.dylib " + f"lib{onedal_lib}.{major_version}.dylib " + f"{ext_lib}".split(" "), + shell=False, + ) class develop(orig_develop.develop, custom_build): @@ -403,12 +455,12 @@ def run(self): project_urls = { - 'Bug Tracker': 'https://github.com/intel/scikit-learn-intelex', - 'Documentation': 'https://intelpython.github.io/daal4py/', - 'Source Code': 'https://github.com/intel/scikit-learn-intelex/daal4py' + "Bug Tracker": "https://github.com/intel/scikit-learn-intelex", + "Documentation": "https://intelpython.github.io/daal4py/", + "Source Code": "https://github.com/intel/scikit-learn-intelex/daal4py", } -with open('README.md', 'r', encoding='utf8') as f: +with open("README.md", "r", encoding="utf8") as f: long_description = f.read() packages_with_tests = [ @@ -438,27 +490,25 @@ def run(self): 'onedal.utils'] if ONEDAL_VERSION >= 20230100: - packages_with_tests += [ - 'onedal.basic_statistics', - 'onedal.linear_model'] + packages_with_tests += ["onedal.basic_statistics", "onedal.linear_model"] if ONEDAL_VERSION >= 20230200: - packages_with_tests += [ - 'onedal.cluster'] + packages_with_tests += ["onedal.cluster"] if build_distribute: packages_with_tests += [ - 'onedal.spmd', - 'onedal.spmd.decomposition', - 'onedal.spmd.ensemble'] + "onedal.spmd", + "onedal.spmd.decomposition", + "onedal.spmd.ensemble", + ] if ONEDAL_VERSION >= 20230100: packages_with_tests += [ - 'onedal.spmd.basic_statistics', - 'onedal.spmd.linear_model', - 'onedal.spmd.neighbors'] + "onedal.spmd.basic_statistics", + "onedal.spmd.linear_model", + "onedal.spmd.neighbors", + ] if ONEDAL_VERSION >= 20230200: - packages_with_tests += [ - 'onedal.spmd.cluster'] + packages_with_tests += ["onedal.spmd.cluster"] setup( name="daal4py", @@ -468,52 +518,47 @@ def run(self): license="Apache-2.0", author="Intel Corporation", version=d4p_version, - url='https://github.com/intel/scikit-learn-intelex', + url="https://github.com/intel/scikit-learn-intelex", author_email="onedal.maintainers@intel.com", maintainer_email="onedal.maintainers@intel.com", project_urls=project_urls, - cmdclass={'develop': develop, 'build': build, 'build_ext': parallel_build_ext}, + cmdclass={"develop": develop, "build": build, "build_ext": parallel_build_ext}, classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'Intended Audience :: Other Audience', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Other Audience", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - 'Topic :: Scientific/Engineering', - 'Topic :: System', - 'Topic :: Software Development', + "Topic :: Scientific/Engineering", + "Topic :: System", + "Topic :: Software Development", ], - python_requires='>=3.7', + python_requires=">=3.7", install_requires=[ "scikit-learn>=0.24", "numpy>=1.19.5 ; python_version <= '3.9'", "numpy>=1.21.6 ; python_version == '3.10'", - "numpy>=1.23.5 ; python_version >= '3.11'" - ], - keywords=[ - 'machine learning', - 'scikit-learn', - 'data science', - 'data analytics' + "numpy>=1.23.5 ; python_version >= '3.11'", ], + keywords=["machine learning", "scikit-learn", "data science", "data analytics"], packages=get_packages_with_tests(packages_with_tests), package_data={ - 'daal4py.oneapi': [ - 'liboneapi_backend.so', - 'oneapi_backend.lib', - 'oneapi_backend.dll', + "daal4py.oneapi": [ + "liboneapi_backend.so", + "oneapi_backend.lib", + "oneapi_backend.dll", ], - 'onedal': get_onedal_py_libs() + "onedal": get_onedal_py_libs(), }, - ext_modules=getpyexts() + ext_modules=getpyexts(), ) From 8799b63863de966fc1fc3728d77fdbe6d18af858 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 12 Jul 2023 04:21:29 -0700 Subject: [PATCH 4/9] feat: provide pre-commit-config --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..aa0cbd02d5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: + - repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + language_version: python3.10 From 11c4deb45b325eb5f0553787532487bd2c80c3f9 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 12 Jul 2023 04:55:22 -0700 Subject: [PATCH 5/9] feat: add isort to lint workflow --- .github/workflows/{black.yml => lint.yml} | 10 ++++++++++ .pre-commit-config.yaml | 5 +++++ 2 files changed, 15 insertions(+) rename .github/workflows/{black.yml => lint.yml} (69%) diff --git a/.github/workflows/black.yml b/.github/workflows/lint.yml similarity index 69% rename from .github/workflows/black.yml rename to .github/workflows/lint.yml index 606f29a91e..7e3cf37b7e 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/lint.yml @@ -18,6 +18,16 @@ jobs: else echo "HAS_CHANGES=true" >> "$GITHUB_OUTPUT" fi + - uses: actions/setup-python@v4 + name: Setup python for isort + if: steps.changedfiles.outputs.HAS_CHANGES == 'true' + with: + python-version: "3.10" + - name: Run isort on changed files + if: steps.changedfiles.outputs.HAS_CHANGES == 'true' + run: | + python -m pip install isort + isort --check ${{ steps.changedfiles.outputs.CHANGED_FILES }} - uses: psf/black@stable name: Run black on changed files if: steps.changedfiles.outputs.HAS_CHANGES == 'true' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa0cbd02d5..522e2eedcd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,3 +4,8 @@ repos: hooks: - id: black language_version: python3.10 + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + language_version: python3.10 From 119da1f5673d3849821afa9ef282d905e4de5d81 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 12 Jul 2023 05:02:43 -0700 Subject: [PATCH 6/9] isort setup.py / test pre-commit hooks --- pyproject.toml | 4 ++++ setup.py | 23 +++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3723d2ede3..290e5f2056 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,3 +19,7 @@ line-length = 90 target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] extend-ignore = 'E203' + +[tool.isort] +profile = "black" +line_length = 90 diff --git a/setup.py b/setup.py index 6f0a24ee9c..a85f820812 100644 --- a/setup.py +++ b/setup.py @@ -15,24 +15,27 @@ # limitations under the License. # =============================================================================== +import distutils.command.build as orig_build +import glob + # System imports import os +import pathlib import sys import time -from setuptools import setup, Extension -from setuptools.command.build_ext import build_ext as _build_ext -import setuptools.command.develop as orig_develop -import distutils.command.build as orig_build -from os.path import join as jp -import pathlib +from concurrent.futures import ThreadPoolExecutor from distutils.sysconfig import get_config_vars -from Cython.Build import cythonize -import glob +from os.path import join as jp + import numpy as np -from scripts.version import get_onedal_version +import setuptools.command.develop as orig_develop +from Cython.Build import cythonize +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext as _build_ext + import scripts.build_backend as build_backend from scripts.package_helpers import get_packages_with_tests -from concurrent.futures import ThreadPoolExecutor +from scripts.version import get_onedal_version try: from ctypes.utils import find_library From 8bf55bbe38636685c1d2b0fa2f67bcd065eccc4c Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 14 Jul 2023 00:19:50 -0700 Subject: [PATCH 7/9] fix: reset setup.py to origin/master to resolve conflicts --- setup.py | 494 +++++++++++++++++++++++++------------------------------ 1 file changed, 223 insertions(+), 271 deletions(-) diff --git a/setup.py b/setup.py index a85f820812..8ee45bb2a9 100644 --- a/setup.py +++ b/setup.py @@ -15,27 +15,24 @@ # limitations under the License. # =============================================================================== -import distutils.command.build as orig_build -import glob - # System imports import os -import pathlib import sys import time -from concurrent.futures import ThreadPoolExecutor -from distutils.sysconfig import get_config_vars -from os.path import join as jp - -import numpy as np +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as _build_ext import setuptools.command.develop as orig_develop +import distutils.command.build as orig_build +from os.path import join as jp +import pathlib +from distutils.sysconfig import get_config_vars from Cython.Build import cythonize -from setuptools import Extension, setup -from setuptools.command.build_ext import build_ext as _build_ext - +import glob +import numpy as np +from scripts.version import get_onedal_version import scripts.build_backend as build_backend from scripts.package_helpers import get_packages_with_tests -from scripts.version import get_onedal_version +from concurrent.futures import ThreadPoolExecutor try: from ctypes.utils import find_library @@ -46,65 +43,60 @@ IS_MAC = False IS_LIN = False -dal_root = os.environ.get("DALROOT") +dal_root = os.environ.get('DALROOT') if dal_root is None: raise RuntimeError("Not set DALROOT variable") -if "linux" in sys.platform: +if 'linux' in sys.platform: IS_LIN = True - lib_dir = jp(dal_root, "lib", "intel64") -elif sys.platform == "darwin": + lib_dir = jp(dal_root, 'lib', 'intel64') +elif sys.platform == 'darwin': IS_MAC = True - lib_dir = jp(dal_root, "lib") -elif sys.platform in ["win32", "cygwin"]: + lib_dir = jp(dal_root, 'lib') +elif sys.platform in ['win32', 'cygwin']: IS_WIN = True - lib_dir = jp(dal_root, "lib", "intel64") + lib_dir = jp(dal_root, 'lib', 'intel64') else: - assert False, sys.platform + " not supported" + assert False, sys.platform + ' not supported' ONEDAL_MAJOR_BINARY_VERSION, ONEDAL_MINOR_BINARY_VERSION = get_onedal_version( - dal_root, "binary" -) + dal_root, 'binary') ONEDAL_VERSION = get_onedal_version(dal_root) ONEDAL_2021_3 = 2021 * 10000 + 3 * 100 ONEDAL_2023_0_1 = 2023 * 10000 + 0 * 100 + 1 -is_onedal_iface = ( - os.environ.get("OFF_ONEDAL_IFACE") is None and ONEDAL_VERSION >= ONEDAL_2021_3 -) +is_onedal_iface = os.environ.get( + 'OFF_ONEDAL_IFACE') is None and ONEDAL_VERSION >= ONEDAL_2021_3 -d4p_version = ( - os.environ["DAAL4PY_VERSION"] - if "DAAL4PY_VERSION" in os.environ - else time.strftime("%Y%m%d.%H%M%S") -) +d4p_version = (os.environ['DAAL4PY_VERSION'] if 'DAAL4PY_VERSION' in os.environ + else time.strftime('%Y%m%d.%H%M%S')) -trues = ["true", "True", "TRUE", "1", "t", "T", "y", "Y", "Yes", "yes", "YES"] -no_dist = True if "NO_DIST" in os.environ and os.environ["NO_DIST"] in trues else False -no_stream = "NO_STREAM" in os.environ and os.environ["NO_STREAM"] in trues -mpi_root = None if no_dist else os.environ["MPIROOT"] -dpcpp = True if "DPCPPROOT" in os.environ else False -dpcpp_root = None if not dpcpp else os.environ["DPCPPROOT"] +trues = ['true', 'True', 'TRUE', '1', 't', 'T', 'y', 'Y', 'Yes', 'yes', 'YES'] +no_dist = True if 'NO_DIST' in os.environ and os.environ['NO_DIST'] in trues else False +no_stream = 'NO_STREAM' in os.environ and os.environ['NO_STREAM'] in trues +mpi_root = None if no_dist else os.environ['MPIROOT'] +dpcpp = True if 'DPCPPROOT' in os.environ else False +dpcpp_root = None if not dpcpp else os.environ['DPCPPROOT'] try: import dpctl - - dpctl_available = dpctl.__version__ >= "0.14" + dpctl_available = dpctl.__version__ >= '0.14' except ImportError: dpctl_available = False build_distribute = dpcpp and dpctl_available and not no_dist and IS_LIN -daal_lib_dir = lib_dir if (IS_MAC or os.path.isdir(lib_dir)) else os.path.dirname(lib_dir) +daal_lib_dir = lib_dir if (IS_MAC or os.path.isdir( + lib_dir)) else os.path.dirname(lib_dir) ONEDAL_LIBDIRS = [daal_lib_dir] if IS_WIN: ONEDAL_LIBDIRS.append(f"{os.environ.get('CONDA_PREFIX')}/Library/lib") if no_stream: - print("\nDisabling support for streaming mode\n") + print('\nDisabling support for streaming mode\n') if no_dist: - print("\nDisabling support for distributed mode\n") + print('\nDisabling support for distributed mode\n') DIST_CFLAGS = [] DIST_CPPS = [] MPI_INCDIRS = [] @@ -112,147 +104,113 @@ MPI_LIBS = [] MPI_CPPS = [] else: - DIST_CFLAGS = [ - "-D_DIST_", - ] - DIST_CPPS = ["src/transceiver.cpp"] - MPI_INCDIRS = [jp(mpi_root, "include")] - MPI_LIBDIRS = [jp(mpi_root, "lib")] - MPI_LIBNAME = getattr(os.environ, "MPI_LIBNAME", None) + DIST_CFLAGS = ['-D_DIST_', ] + DIST_CPPS = ['src/transceiver.cpp'] + MPI_INCDIRS = [jp(mpi_root, 'include')] + MPI_LIBDIRS = [jp(mpi_root, 'lib')] + MPI_LIBNAME = getattr(os.environ, 'MPI_LIBNAME', None) if MPI_LIBNAME: MPI_LIBS = [MPI_LIBNAME] elif IS_WIN: - if os.path.isfile(jp(mpi_root, "lib", "mpi.lib")): - MPI_LIBS = ["mpi"] - if os.path.isfile(jp(mpi_root, "lib", "impi.lib")): - MPI_LIBS = ["impi"] + if os.path.isfile(jp(mpi_root, 'lib', 'mpi.lib')): + MPI_LIBS = ['mpi'] + if os.path.isfile(jp(mpi_root, 'lib', 'impi.lib')): + MPI_LIBS = ['impi'] assert MPI_LIBS, "Couldn't find MPI library" else: - MPI_LIBS = ["mpi"] - MPI_CPPS = ["src/mpi/mpi_transceiver.cpp"] + MPI_LIBS = ['mpi'] + MPI_CPPS = ['src/mpi/mpi_transceiver.cpp'] def get_sdl_cflags(): if IS_LIN or IS_MAC: - return DIST_CFLAGS + [ - "-fstack-protector-strong", - "-fPIC", - "-D_FORTIFY_SOURCE=2", - "-Wformat", - "-Wformat-security", - "-fno-strict-overflow", - "-fno-delete-null-pointer-checks", - ] + return DIST_CFLAGS + ['-fstack-protector-strong', '-fPIC', + '-D_FORTIFY_SOURCE=2', '-Wformat', + '-Wformat-security', '-fno-strict-overflow', + '-fno-delete-null-pointer-checks'] if IS_WIN: - return DIST_CFLAGS + ["-GS"] + return DIST_CFLAGS + ['-GS'] def get_sdl_ldflags(): if IS_LIN: - return [ - "-Wl,-z,noexecstack,-z,relro,-z,now,-fstack-protector-strong," - "-fno-strict-overflow,-fno-delete-null-pointer-checks,-fwrapv" - ] + return ['-Wl,-z,noexecstack,-z,relro,-z,now,-fstack-protector-strong,' + '-fno-strict-overflow,-fno-delete-null-pointer-checks,-fwrapv'] if IS_MAC: - return [ - "-fstack-protector-strong", - "-fno-strict-overflow", - "-fno-delete-null-pointer-checks", - "-fwrapv", - ] + return ['-fstack-protector-strong', + '-fno-strict-overflow', + '-fno-delete-null-pointer-checks', + '-fwrapv'] if IS_WIN: - return ["-NXCompat", "-DynamicBase"] + return ['-NXCompat', '-DynamicBase'] def get_daal_type_defines(): - daal_type_defines = [ - "DAAL_ALGORITHM_FP_TYPE", - "DAAL_SUMMARY_STATISTICS_TYPE", - "DAAL_DATA_TYPE", - ] - return [(d, "double") for d in daal_type_defines] + daal_type_defines = ['DAAL_ALGORITHM_FP_TYPE', + 'DAAL_SUMMARY_STATISTICS_TYPE', + 'DAAL_DATA_TYPE'] + return [(d, 'double') for d in daal_type_defines] -def get_libs(iface="daal"): +def get_libs(iface='daal'): major_version = ONEDAL_MAJOR_BINARY_VERSION if IS_WIN: - libraries_plat = [f"onedal_core_dll.{major_version}"] - onedal_lib = [f"onedal_dll.{major_version}"] - onedal_dpc_lib = [f"onedal_dpc_dll.{major_version}"] + libraries_plat = [f'onedal_core_dll.{major_version}'] + onedal_lib = [f'onedal_dll.{major_version}'] + onedal_dpc_lib = [f'onedal_dpc_dll.{major_version}'] elif IS_MAC: - libraries_plat = [ - f"onedal_core.{major_version}", - f"onedal_thread.{major_version}", - ] - onedal_lib = [f"onedal.{major_version}"] - onedal_dpc_lib = [f"onedal_dpc.{major_version}"] + libraries_plat = [f'onedal_core.{major_version}', + f'onedal_thread.{major_version}'] + onedal_lib = [f'onedal.{major_version}'] + onedal_dpc_lib = [f'onedal_dpc.{major_version}'] else: - libraries_plat = [ - f":libonedal_core.so.{major_version}", - f":libonedal_thread.so.{major_version}", - ] - onedal_lib = [f":libonedal.so.{major_version}"] - onedal_dpc_lib = [f":libonedal_dpc.so.{major_version}"] - if iface == "onedal": + libraries_plat = [f':libonedal_core.so.{major_version}', + f':libonedal_thread.so.{major_version}'] + onedal_lib = [f':libonedal.so.{major_version}'] + onedal_dpc_lib = [f':libonedal_dpc.so.{major_version}'] + if iface == 'onedal': libraries_plat = onedal_lib + libraries_plat - elif iface == "onedal_dpc": + elif iface == 'onedal_dpc': libraries_plat = onedal_dpc_lib + libraries_plat return libraries_plat def get_build_options(): - include_dir_plat = [ - os.path.abspath("./src"), - os.path.abspath("."), - dal_root + "/include", - ] + include_dir_plat = [os.path.abspath('./src'), + os.path.abspath('.'), + dal_root + '/include', ] # FIXME it is a wrong place for this dependency if not no_dist: - include_dir_plat.append(mpi_root + "/include") - using_intel = os.environ.get("cc", "") in [ - "icc", - "icpc", - "icl", - "dpcpp", - "icx", - "icpx", - ] - eca = [ - "-DPY_ARRAY_UNIQUE_SYMBOL=daal4py_array_API", - '-DD4P_VERSION="' + d4p_version + '"', - "-DNPY_ALLOW_THREADS=1", - ] + include_dir_plat.append(mpi_root + '/include') + using_intel = os.environ.get('cc', '') in [ + 'icc', 'icpc', 'icl', 'dpcpp', 'icx', 'icpx'] + eca = ['-DPY_ARRAY_UNIQUE_SYMBOL=daal4py_array_API', + '-DD4P_VERSION="' + d4p_version + '"', '-DNPY_ALLOW_THREADS=1'] ela = [] if using_intel and IS_WIN: include_dir_plat.append( - jp(os.environ.get("ICPP_COMPILER16", ""), "compiler", "include") - ) - eca += ["-std=c++17", "-w", "/MD"] + jp(os.environ.get('ICPP_COMPILER16', ''), 'compiler', 'include')) + eca += ['-std=c++17', '-w', '/MD'] elif not using_intel and IS_WIN: - eca += ["-wd4267", "-wd4244", "-wd4101", "-wd4996", "/std:c++17"] + eca += ['-wd4267', '-wd4244', '-wd4101', '-wd4996', '/std:c++17'] else: - eca += [ - "-std=c++17", - "-w", - ] # '-D_GLIBCXX_USE_CXX11_ABI=0'] + eca += ['-std=c++17', '-w', ] # '-D_GLIBCXX_USE_CXX11_ABI=0'] # Security flags eca += get_sdl_cflags() ela += get_sdl_ldflags() if IS_MAC: - eca.append("-stdlib=libc++") - ela.append("-stdlib=libc++") + eca.append('-stdlib=libc++') + ela.append('-stdlib=libc++') ela.append("-Wl,-rpath,{}".format(daal_lib_dir)) ela.append("-Wl,-rpath,@loader_path/../../../") elif IS_WIN: - ela.append("-IGNORE:4197") - elif IS_LIN and not any( - x in os.environ and "-g" in os.environ[x] - for x in ["CPPFLAGS", "CFLAGS", "LDFLAGS"] - ): - ela.append("-s") + ela.append('-IGNORE:4197') + elif IS_LIN and not any(x in os.environ and '-g' in os.environ[x] + for x in ['CPPFLAGS', 'CFLAGS', 'LDFLAGS']): + ela.append('-s') if IS_LIN: ela.append("-fPIC") ela.append("-Wl,-rpath,$ORIGIN/../../../") @@ -265,23 +223,18 @@ def getpyexts(): exts = [] - ext = Extension( - "daal4py._daal4py", - [ - os.path.abspath("src/daal4py.cpp"), - os.path.abspath("build/daal4py_cpp.cpp"), - os.path.abspath("build/daal4py_cy.pyx"), - ] - + DIST_CPPS, - depends=glob.glob(jp(os.path.abspath("src"), "*.h")), - include_dirs=include_dir_plat + [np.get_include()], - extra_compile_args=eca, - define_macros=get_daal_type_defines(), - extra_link_args=ela, - libraries=libraries_plat, - library_dirs=ONEDAL_LIBDIRS, - language="c++", - ) + ext = Extension('daal4py._daal4py', + [os.path.abspath('src/daal4py.cpp'), + os.path.abspath('build/daal4py_cpp.cpp'), + os.path.abspath('build/daal4py_cy.pyx')] + DIST_CPPS, + depends=glob.glob(jp(os.path.abspath('src'), '*.h')), + include_dirs=include_dir_plat + [np.get_include()], + extra_compile_args=eca, + define_macros=get_daal_type_defines(), + extra_link_args=ela, + libraries=libraries_plat, + library_dirs=ONEDAL_LIBDIRS, + language='c++') exts.extend(cythonize(ext)) if dpcpp: @@ -290,111 +243,105 @@ def getpyexts(): elif IS_WIN: runtime_oneapi_dirs = [] - ext = Extension( - "daal4py._oneapi", - [ - os.path.abspath("src/oneapi/oneapi.pyx"), - ], - depends=["src/oneapi/oneapi.h", "src/oneapi/oneapi_backend.h"], - include_dirs=include_dir_plat + [np.get_include()], - extra_compile_args=eca, - extra_link_args=ela, - define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")], - libraries=["oneapi_backend"] + libraries_plat, - library_dirs=["daal4py/oneapi"] + ONEDAL_LIBDIRS, - runtime_library_dirs=runtime_oneapi_dirs, - language="c++", - ) + ext = Extension('daal4py._oneapi', + [os.path.abspath('src/oneapi/oneapi.pyx'), ], + depends=['src/oneapi/oneapi.h', 'src/oneapi/oneapi_backend.h'], + include_dirs=include_dir_plat + [np.get_include()], + extra_compile_args=eca, + extra_link_args=ela, + define_macros=[ + ('NPY_NO_DEPRECATED_API', + 'NPY_1_7_API_VERSION') + ], + libraries=['oneapi_backend'] + libraries_plat, + library_dirs=['daal4py/oneapi'] + ONEDAL_LIBDIRS, + runtime_library_dirs=runtime_oneapi_dirs, + language='c++') exts.extend(cythonize(ext)) if not no_dist: mpi_include_dir = include_dir_plat + [np.get_include()] + MPI_INCDIRS - mpi_depens = glob.glob(jp(os.path.abspath("src"), "*.h")) + mpi_depens = glob.glob(jp(os.path.abspath('src'), '*.h')) mpi_extra_link = ela + ["-Wl,-rpath,{}".format(x) for x in MPI_LIBDIRS] - exts.append( - Extension( - "daal4py.mpi_transceiver", - MPI_CPPS, - depends=mpi_depens, - include_dirs=mpi_include_dir, - extra_compile_args=eca, - define_macros=get_daal_type_defines(), - extra_link_args=mpi_extra_link, - libraries=libraries_plat + MPI_LIBS, - library_dirs=ONEDAL_LIBDIRS + MPI_LIBDIRS, - language="c++", - ) - ) + exts.append(Extension('daal4py.mpi_transceiver', + MPI_CPPS, + depends=mpi_depens, + include_dirs=mpi_include_dir, + extra_compile_args=eca, + define_macros=get_daal_type_defines(), + extra_link_args=mpi_extra_link, + libraries=libraries_plat + MPI_LIBS, + library_dirs=ONEDAL_LIBDIRS + MPI_LIBDIRS, + language='c++')) return exts cfg_vars = get_config_vars() for key, value in get_config_vars().items(): if isinstance(value, str): - cfg_vars[key] = value.replace("-Wstrict-prototypes", "").replace("-DNDEBUG", "") + cfg_vars[key] = value.replace( + "-Wstrict-prototypes", "").replace('-DNDEBUG', '') def gen_pyx(odir): - gtr_files = glob.glob(jp(os.path.abspath("generator"), "*")) + ["./setup.py"] - src_files = [ - os.path.abspath("build/daal4py_cpp.h"), - os.path.abspath("build/daal4py_cpp.cpp"), - os.path.abspath("build/daal4py_cy.pyx"), - ] + gtr_files = glob.glob( + jp(os.path.abspath('generator'), '*')) + ['./setup.py'] + src_files = [os.path.abspath('build/daal4py_cpp.h'), + os.path.abspath('build/daal4py_cpp.cpp'), + os.path.abspath('build/daal4py_cy.pyx')] if all(os.path.isfile(x) for x in src_files): src_files.sort(key=os.path.getmtime) gtr_files.sort(key=os.path.getmtime, reverse=True) if os.path.getmtime(src_files[0]) > os.path.getmtime(gtr_files[0]): - print( - "Generated files are all newer than generator code." - "Skipping code generation" - ) + print('Generated files are all newer than generator code.' + 'Skipping code generation') return from generator.gen_daal4py import gen_daal4py - odir = os.path.abspath(odir) if not os.path.isdir(odir): os.mkdir(odir) - gen_daal4py(dal_root, odir, d4p_version, no_dist=no_dist, no_stream=no_stream) + gen_daal4py(dal_root, odir, d4p_version, + no_dist=no_dist, no_stream=no_stream) -gen_pyx(os.path.abspath("./build")) +gen_pyx(os.path.abspath('./build')) def build_oneapi_backend(): eca, ela, includes = get_build_options() - cc = "icx" + cc = 'icx' if IS_WIN: - cxx = "icx" + cxx = 'icx' else: - cxx = "icpx" - eca = ["-fsycl"] + ["-fsycl-device-code-split=per_kernel"] + eca - ela = ["-fsycl"] + ["-fsycl-device-code-split=per_kernel"] + ela + cxx = 'icpx' + eca = ['-fsycl'] + ['-fsycl-device-code-split=per_kernel'] + eca + ela = ['-fsycl'] + ['-fsycl-device-code-split=per_kernel'] + ela return build_backend.build_cpp( cc=cc, cxx=cxx, - sources=["src/oneapi/oneapi_backend.cpp"], - targetname="oneapi_backend", - targetprefix="" if IS_WIN else "lib", - targetsuffix=".dll" if IS_WIN else ".so", - libs=get_libs("daal") + ["OpenCL", "onedal_sycl"], + sources=['src/oneapi/oneapi_backend.cpp'], + targetname='oneapi_backend', + targetprefix='' if IS_WIN else 'lib', + targetsuffix='.dll' if IS_WIN else '.so', + libs=get_libs('daal') + ['OpenCL', 'onedal_sycl'], libdirs=ONEDAL_LIBDIRS, includes=includes, eca=eca, ela=ela, defines=[], - installpath="daal4py/oneapi/", + installpath='daal4py/oneapi/' ) def get_onedal_py_libs(): - ext_suffix = get_config_vars("EXT_SUFFIX")[0] - libs = [f"_onedal_py_host{ext_suffix}", f"_onedal_py_dpc{ext_suffix}"] + ext_suffix = get_config_vars('EXT_SUFFIX')[0] + libs = [f'_onedal_py_host{ext_suffix}', f'_onedal_py_dpc{ext_suffix}'] if IS_WIN: - ext_suffix_lib = ext_suffix.replace(".dll", ".lib") - libs += [f"_onedal_py_host{ext_suffix_lib}", f"_onedal_py_dpc{ext_suffix_lib}"] + ext_suffix_lib = ext_suffix.replace('.dll', '.lib') + libs += [f'_onedal_py_host{ext_suffix_lib}', + f'_onedal_py_dpc{ext_suffix_lib}'] return libs @@ -406,41 +353,39 @@ def build_extensions(self): executor.submit(self.build_extension, ext) -class custom_build: +class custom_build(): def run(self): if is_onedal_iface: - cxx = os.getenv("CXX", "cl" if IS_WIN else "g++") + cxx = os.getenv('CXX', 'cl' if IS_WIN else 'g++') build_backend.custom_build_cmake_clib( - "host", cxx, ONEDAL_MAJOR_BINARY_VERSION, no_dist=no_dist - ) + 'host', cxx, ONEDAL_MAJOR_BINARY_VERSION, no_dist=no_dist) if dpcpp: build_oneapi_backend() if is_onedal_iface: build_backend.custom_build_cmake_clib( - "dpc", ONEDAL_MAJOR_BINARY_VERSION, no_dist=no_dist - ) + 'dpc', ONEDAL_MAJOR_BINARY_VERSION, no_dist=no_dist) def post_build(self): if IS_MAC: import subprocess - # manually fix incorrect install_name of oneDAL 2023.0.1 libs major_version = ONEDAL_MAJOR_BINARY_VERSION - major_is_available = ( - find_library(f"libonedal_core.{major_version}.dylib") is not None - ) + major_is_available = find_library( + f'libonedal_core.{major_version}.dylib') is not None if major_is_available and ONEDAL_VERSION == ONEDAL_2023_0_1: - extension_libs = list(pathlib.Path(".").glob("**/*darwin.so")) - onedal_libs = ["onedal", "onedal_dpc", "onedal_core", "onedal_thread"] + extension_libs = list(pathlib.Path('.').glob('**/*darwin.so')) + onedal_libs = [ + 'onedal', + 'onedal_dpc', + 'onedal_core', + 'onedal_thread'] for ext_lib in extension_libs: for onedal_lib in onedal_libs: subprocess.call( - "/usr/bin/install_name_tool -change " - f"lib{onedal_lib}.dylib " - f"lib{onedal_lib}.{major_version}.dylib " - f"{ext_lib}".split(" "), - shell=False, - ) + '/usr/bin/install_name_tool -change ' + f'lib{onedal_lib}.dylib ' + f'lib{onedal_lib}.{major_version}.dylib ' + f'{ext_lib}'.split(' '), shell=False) class develop(orig_develop.develop, custom_build): @@ -458,12 +403,12 @@ def run(self): project_urls = { - "Bug Tracker": "https://github.com/intel/scikit-learn-intelex", - "Documentation": "https://intelpython.github.io/daal4py/", - "Source Code": "https://github.com/intel/scikit-learn-intelex/daal4py", + 'Bug Tracker': 'https://github.com/intel/scikit-learn-intelex', + 'Documentation': 'https://intelpython.github.io/daal4py/', + 'Source Code': 'https://github.com/intel/scikit-learn-intelex/daal4py' } -with open("README.md", "r", encoding="utf8") as f: +with open('README.md', 'r', encoding='utf8') as f: long_description = f.read() packages_with_tests = [ @@ -493,25 +438,27 @@ def run(self): 'onedal.utils'] if ONEDAL_VERSION >= 20230100: - packages_with_tests += ["onedal.basic_statistics", "onedal.linear_model"] + packages_with_tests += [ + 'onedal.basic_statistics', + 'onedal.linear_model'] if ONEDAL_VERSION >= 20230200: - packages_with_tests += ["onedal.cluster"] + packages_with_tests += [ + 'onedal.cluster'] if build_distribute: packages_with_tests += [ - "onedal.spmd", - "onedal.spmd.decomposition", - "onedal.spmd.ensemble", - ] + 'onedal.spmd', + 'onedal.spmd.decomposition', + 'onedal.spmd.ensemble'] if ONEDAL_VERSION >= 20230100: packages_with_tests += [ - "onedal.spmd.basic_statistics", - "onedal.spmd.linear_model", - "onedal.spmd.neighbors", - ] + 'onedal.spmd.basic_statistics', + 'onedal.spmd.linear_model', + 'onedal.spmd.neighbors'] if ONEDAL_VERSION >= 20230200: - packages_with_tests += ["onedal.spmd.cluster"] + packages_with_tests += [ + 'onedal.spmd.cluster'] setup( name="daal4py", @@ -521,47 +468,52 @@ def run(self): license="Apache-2.0", author="Intel Corporation", version=d4p_version, - url="https://github.com/intel/scikit-learn-intelex", + url='https://github.com/intel/scikit-learn-intelex', author_email="onedal.maintainers@intel.com", maintainer_email="onedal.maintainers@intel.com", project_urls=project_urls, - cmdclass={"develop": develop, "build": build, "build_ext": parallel_build_ext}, + cmdclass={'develop': develop, 'build': build, 'build_ext': parallel_build_ext}, classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "Intended Audience :: Other Audience", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Other Audience', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "Topic :: Scientific/Engineering", - "Topic :: System", - "Topic :: Software Development", + 'Topic :: Scientific/Engineering', + 'Topic :: System', + 'Topic :: Software Development', ], - python_requires=">=3.7", + python_requires='>=3.7', install_requires=[ "scikit-learn>=0.24", "numpy>=1.19.5 ; python_version <= '3.9'", "numpy>=1.21.6 ; python_version == '3.10'", - "numpy>=1.23.5 ; python_version >= '3.11'", + "numpy>=1.23.5 ; python_version >= '3.11'" + ], + keywords=[ + 'machine learning', + 'scikit-learn', + 'data science', + 'data analytics' ], - keywords=["machine learning", "scikit-learn", "data science", "data analytics"], packages=get_packages_with_tests(packages_with_tests), package_data={ - "daal4py.oneapi": [ - "liboneapi_backend.so", - "oneapi_backend.lib", - "oneapi_backend.dll", + 'daal4py.oneapi': [ + 'liboneapi_backend.so', + 'oneapi_backend.lib', + 'oneapi_backend.dll', ], - "onedal": get_onedal_py_libs(), + 'onedal': get_onedal_py_libs() }, - ext_modules=getpyexts(), + ext_modules=getpyexts() ) From 013edfa9890516d4420c71915dc4de783539e36a Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 20 Jul 2023 05:26:24 -0700 Subject: [PATCH 8/9] add copyright info --- .pre-commit-config.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 522e2eedcd..69b711d5ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,19 @@ +#=============================================================================== +# Copyright 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + repos: - repo: https://github.com/psf/black rev: 23.7.0 From 0a3f5560fc26fb448e0da5958e2619c83ab84e3e Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 20 Jul 2023 06:23:53 -0700 Subject: [PATCH 9/9] use set -x in run commands --- .github/workflows/lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7e3cf37b7e..2b766d4e56 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,6 +12,7 @@ jobs: - id: changedfiles name: Get list of changed .py files run: | + set -x echo "CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep .py$ | xargs)" >> "$GITHUB_OUTPUT" if [[ $(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep .py$ | wc -l) -eq 0 ]]; then echo "HAS_CHANGES=false" >> "$GITHUB_OUTPUT" @@ -26,6 +27,7 @@ jobs: - name: Run isort on changed files if: steps.changedfiles.outputs.HAS_CHANGES == 'true' run: | + set -x python -m pip install isort isort --check ${{ steps.changedfiles.outputs.CHANGED_FILES }} - uses: psf/black@stable