From a70e5b372b8e1aadd0ab7e5de15b1dd0ba1b8818 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 20 Nov 2023 14:24:52 -0700 Subject: [PATCH 01/53] first attempt at meson building --- meson.build | 50 +++++++++++++++++++ meson.options | 1 + pydiso/meson.build | 82 ++++++++++++++++++++++++++++++++ pydiso/mkl_solver.pyx | 22 +++++---- pyproject.toml | 108 ++++++++++++++++++++++++++++++++++++++++++ setup.py | 70 --------------------------- 6 files changed, 255 insertions(+), 78 deletions(-) create mode 100644 meson.build create mode 100644 meson.options create mode 100644 pydiso/meson.build create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..3303d66 --- /dev/null +++ b/meson.build @@ -0,0 +1,50 @@ +project( + 'pydiso', + 'c', 'cython', + # Note that the git commit hash cannot be added dynamically here + # (it is dynamically generated though setuptools_scm) + version: run_command('python', + [ + '-c', + ''' +from setuptools_scm import get_version +print(get_version())''' + ], + check: true +).stdout().strip(), + + license: 'MIT', + meson_version: '>= 1.1.0', + default_options: [ + 'buildtype=debugoptimized', + 'b_ndebug=if-release', + ], +) + +# https://mesonbuild.com/Python-module.html +py_mod = import('python') +py = py_mod.find_installation(pure: false) +py_dep = py.dependency() + +cc = meson.get_compiler('c') +cy = meson.get_compiler('cython') +# generator() doesn't accept compilers, only found programs - cast it. +cython = find_program(cy.cmd_array()[0]) + +_global_c_args = cc.get_supported_arguments( + '-Wno-unused-but-set-variable', + '-Wno-unused-function', + '-Wno-conversion', + '-Wno-misleading-indentation', +) +add_project_arguments(_global_c_args, language : 'c') + +# We need -lm for all C code (assuming it uses math functions, which is safe to +# assume for SciPy). For C++ it isn't needed, because libstdc++/libc++ is +# guaranteed to depend on it. +m_dep = cc.find_library('m', required : false) +if m_dep.found() + add_project_link_arguments('-lm', language : 'c') +endif + +subdir('pydiso') \ No newline at end of file diff --git a/meson.options b/meson.options new file mode 100644 index 0000000..e39a447 --- /dev/null +++ b/meson.options @@ -0,0 +1 @@ +option('cy_line_trace', type : 'boolean', value : false) \ No newline at end of file diff --git a/pydiso/meson.build b/pydiso/meson.build new file mode 100644 index 0000000..3fe0aaf --- /dev/null +++ b/pydiso/meson.build @@ -0,0 +1,82 @@ +# NumPy include directory +# The try-except is needed because when things are +# split across drives on Windows, there is no relative path and an exception +# gets raised. There may be other such cases, so add a catch-all and switch to +# an absolute path. Relative paths are needed when for example a virtualenv is +# placed inside the source tree; Meson rejects absolute paths to places inside +# the source tree. +# For cross-compilation it is often not possible to run the Python interpreter +# in order to retrieve numpy's include directory. It can be specified in the +# cross file instead: +# [properties] +# numpy-include-dir = /abspath/to/host-pythons/site-packages/numpy/core/include +# +# This uses the path as is, and avoids running the interpreter. +incdir_numpy = meson.get_external_property('numpy-include-dir', 'not-given') +if incdir_numpy == 'not-given' + incdir_numpy = run_command(py, + [ + '-c', + '''import os +import numpy as np +try: + incdir = os.path.relpath(np.get_include()) +except Exception: + incdir = np.get_include() +print(incdir) + ''' + ], + check: true + ).stdout().strip() +else + _incdir_numpy_abs = incdir_numpy +endif +inc_np = include_directories(incdir_numpy) +np_dep = declare_dependency(include_directories: inc_np) + +# find mkl + +mkl_dep = dependency('mkl-sdl', required: true) + +# Deal with M_PI & friends; add `use_math_defines` to c_args or cpp_args +# Cython doesn't always get this right itself (see, e.g., gh-16800), so +# explicitly add the define as a compiler flag for Cython-generated code. +is_windows = host_machine.system() == 'windows' +if is_windows + use_math_defines = ['-D_USE_MATH_DEFINES'] +else + use_math_defines = [] +endif + +numpy_nodepr_api = '-DNPY_NO_DEPRECATED_API=NPY_1_9_API_VERSION' +c_undefined_ok = ['-Wno-maybe-uninitialized'] + +cython_c_args = [numpy_nodepr_api, use_math_defines] + +cy_line_trace = get_option('cy_line_trace') +if cy_line_trace + cython_c_args += ['-DCYTHON_TRACE_NOGIL=1'] +endif + +cython_cpp_args = cython_c_args + +module_path = 'pydiso' + +py.extension_module( + 'mkl_solver', + 'mkl_solver.pyx', + include_directories: incdir_numpy, + c_args: cython_c_args, + install: true, + subdir: module_path, + dependencies : [py_dep, np_dep, mkl_dep], +) + +python_sources = [ + '__init__.py', +] + +py.install_sources( + python_sources, + subdir: module_path +) \ No newline at end of file diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index e1d8c7f..8473802 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -261,7 +261,7 @@ cdef class MKLPardisoSolver: self.shape = n_row, n_col # allocate the lock - self.lock = PyThread_allocate_lock() + #self.lock = PyThread_allocate_lock() self._data_type = A.dtype if matrix_type is None: @@ -296,6 +296,7 @@ cdef class MKLPardisoSolver: #set integer length integer_len = A.indices.itemsize self._is_32 = integer_len == sizeof(int_t) + print("here1") if self._is_32: self._par = _PardisoParams() self._initialize(self._par, A, matrix_type, verbose) @@ -305,18 +306,23 @@ cdef class MKLPardisoSolver: else: raise PardisoError("Unrecognized integer length") self._initialized = True + print("here2") if(verbose): #for reporting factorization progress via python's `print` mkl_set_progress(mkl_progress) else: mkl_set_progress(mkl_no_progress) + print("here3") self._set_A(A.data) + print("here4") self._analyze() + print("here5") self._factored = False if factor: self._factor() + print("here6") def refactor(self, A): """solver.refactor(A) @@ -506,7 +512,7 @@ cdef class MKLPardisoSolver: cdef long_t phase64=-1, nrhs64=0, error64=0 if self._initialized: - PyThread_acquire_lock(self.lock, 1) + #PyThread_acquire_lock(self.lock, 1) if self._is_32: pardiso( self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, @@ -519,12 +525,12 @@ cdef class MKLPardisoSolver: &phase64, &self._par64.n, self.a, NULL, NULL, NULL, &nrhs64, self._par64.iparm, &self._par64.msglvl, NULL, NULL, &error64 ) - PyThread_release_lock(self.lock) + #PyThread_release_lock(self.lock) err = error or error64 if err!=0: raise PardisoError("Memmory release error "+_err_messages[err]) - #dealloc lock - PyThread_free_lock(self.lock) + #dealloc lock + #PyThread_free_lock(self.lock) cdef _analyze(self): #phase = 11 @@ -554,16 +560,16 @@ cdef class MKLPardisoSolver: cdef int_t error=0 cdef long_t error64=0, phase64=phase, nrhs64=nrhs - PyThread_acquire_lock(self.lock, 1) + #PyThread_acquire_lock(self.lock, 1) if self._is_32: pardiso(self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, &phase, &self._par.n, self.a, &self._par.ia[0], &self._par.ja[0], &self._par.perm[0], &nrhs, self._par.iparm, &self._par.msglvl, b, x, &error) - PyThread_release_lock(self.lock) + #PyThread_release_lock(self.lock) return error else: pardiso_64(self.handle, &self._par64.maxfct, &self._par64.mnum, &self._par64.mtype, &phase64, &self._par64.n, self.a, &self._par64.ia[0], &self._par64.ja[0], &self._par64.perm[0], &nrhs64, self._par64.iparm, &self._par64.msglvl, b, x, &error64) - PyThread_release_lock(self.lock) + #PyThread_release_lock(self.lock) return error64 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e93252c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,108 @@ + +[build-system] +build-backend = 'mesonpy' +requires = [ + "meson-python>=0.14.0", + "Cython>=0.29.35", # when updating version, also update check in meson.build + "setuptools_scm[toml]>=6.2", + + # This package automatically provides all of the numpy pinning for different python + # versions and runtime requirements. + "oldest-supported-numpy", + + # The following is taken from scipy's pyproject.toml file to handle + # building against the proper numpy API + + # When numpy 2.0.0rc1 comes out, we should update this to build against 2.0, + # and then runtime depend on the range 1.22.X to <2.3. No need to switch to + # 1.25.2 in the meantime (1.25.x is the first version which exports older C + # API versions by default). + + # default numpy requirements + # "numpy==1.22.4; python_version<='3.10' and platform_python_implementation != 'PyPy'", + # "numpy==1.23.2; python_version=='3.11' and platform_python_implementation != 'PyPy'", + + # For Python versions which aren't yet officially supported, we specify an + # unpinned NumPy which allows source distributions to be used and allows + # wheels to be used as soon as they become available. + # "numpy>=1.26.0b1; python_version>='3.12'", + # "numpy; python_version>='3.8' and platform_python_implementation=='PyPy'", +] + +[project] +name = 'pydiso' +dynamic = ["version"] +description = "Wrapper for intel's pardiso implementation in the MKL" +readme = 'README.md' +requires-python = '>=3.8' +authors = [ + {name = 'SimPEG developers', email = 'josephrcapriotti@gmail.com'}, +] +keywords = [ + 'sparse', 'solver', 'wrapper', +] + +# Note: Python and NumPy upper version bounds should be set correctly in +# release branches, see: +# https://scipy.github.io/devdocs/dev/core-dev/index.html#version-ranges-for-numpy-and-other-dependencies +dependencies = [ + # TODO: update to "pin-compatible" once possible, see + # https://github.com/mesonbuild/meson-python/issues/29 + "numpy>=1.22.4", + "scipy>=1.8", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Cython", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Physics", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Operating System :: Unix", + "Operating System :: MacOS", + "Natural Language :: English", +] + +[project.license] +file = 'LICENSE' + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-cov", +] + +build = [ + "meson-python>=0.14.0", + "meson", + "ninja", + "numpy>=1.22.4", + "cython>=0.29.35", + "setuptools_scm", +] + +[project.urls] +Homepage = 'https://simpeg.xyz' +#Documentation = 'https://discretize.simpeg.xyz' +Repository = 'https://github.com/simpeg/pydiso.git' + +[tool.setuptools_scm] + +[tool.meson-python.args] +# for windows this will prioritize the msvc compilers +setup = ['--vsenv'] + +[tool.cibuildwheel] +# skip building wheels for python 3.6, 3.7, all pypy versions, and specialty linux +# processors (still does arm builds though). +# skip windows 32bit +skip = "cp36-* cp37-* pp* *_ppc64le *_i686 *_s390x *-win32 cp38-musllinux_x86_64" +build-verbosity = "3" + +# test importing discretize to make sure externals are loadable. +test-command = 'python -c "import pydiso; print(pydiso.__version__)"' \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index ac83b4c..0000000 --- a/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -from distutils.core import setup -from setuptools import find_packages - -import sys - -def configuration(parent_package="", top_path=None): - from numpy.distutils.misc_util import Configuration - - config = Configuration(None, parent_package, top_path) - config.set_options( - ignore_setup_xxx_py=True, - assume_default_configuration=True, - delegate_options_to_subpackages=True, - quiet=True, - ) - - config.add_subpackage("pydiso") - - return config - -metadata = dict( - name='pydiso', - version='0.0.3', - python_requires=">=3.8", - setup_requires=[ - "numpy>=1.8", - "cython>=3.0", - ], - install_requires=[ - 'numpy>=1.8', - 'scipy>=0.13', - ], - author='SimPEG developers', - author_email='josephrcapriotti@gmail.com', - description="Wrapper for intel's pardiso implementation in the MKL", - keywords='sparse, solver, wrapper', - url='https://www.simpeg.xyz', - download_url='https://github.com/jcapriot/pydiso-mkl', - platforms=['Windows', 'Linux', 'Solaris', 'Mac OS-X', 'Unix'], - license='MIT License' -) - -if len(sys.argv) >= 2 and ( - "--help" in sys.argv[1:] - or sys.argv[1] in ("--help-commands", "egg_info", "--version", "clean") -): - # For these actions, NumPy is not required. - # - # They are required to succeed without Numpy, for example when - # pip is used to install discretize when Numpy is not yet present in - # the system. - try: - from setuptools import setup - except ImportError: - from distutils.core import setup -else: - if (len(sys.argv) >= 2 and sys.argv[1] in ("bdist_wheel", "bdist_egg")) or ( - "develop" in sys.argv - ): - # bdist_wheel/bdist_egg needs setuptools - import setuptools - - from numpy.distutils.core import setup - - # Add the configuration to the setup dict when building - # after numpy is installed - metadata["configuration"] = configuration - - -setup(**metadata) From c0eee6c12b30a0c421e4fb794ad7863ed3d47fa3 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Tue, 21 Nov 2023 16:33:12 -0700 Subject: [PATCH 02/53] include mkl-devel in build requirements --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e93252c..83ac415 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,8 @@ requires = [ # This package automatically provides all of the numpy pinning for different python # versions and runtime requirements. "oldest-supported-numpy", + # oldest mkl-devel available (for building against). + "mkl-devel==2021.1.1" # The following is taken from scipy's pyproject.toml file to handle # building against the proper numpy API From a12b70c5b1e15676283df6a90505b3497614c059 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Tue, 21 Nov 2023 16:48:17 -0700 Subject: [PATCH 03/53] change version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 83ac415..a183045 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ requires = [ # versions and runtime requirements. "oldest-supported-numpy", # oldest mkl-devel available (for building against). - "mkl-devel==2021.1.1" + "mkl-devel==2023.0.0" # The following is taken from scipy's pyproject.toml file to handle # building against the proper numpy API From fd02e86eecb6682aa803b841ed382ac48c7ce6b8 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Tue, 21 Nov 2023 16:52:49 -0700 Subject: [PATCH 04/53] relax version a bit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a183045..1645e8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ requires = [ # versions and runtime requirements. "oldest-supported-numpy", # oldest mkl-devel available (for building against). - "mkl-devel==2023.0.0" + "mkl-devel==2023" # The following is taken from scipy's pyproject.toml file to handle # building against the proper numpy API From c10396c8e6300ff0908da0ff3b6ee6f927bb25ff Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Tue, 21 Nov 2023 17:02:26 -0700 Subject: [PATCH 05/53] use examples from numpy --- meson.options | 8 +++++++- pydiso/meson.build | 32 +++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/meson.options b/meson.options index e39a447..1ae0e79 100644 --- a/meson.options +++ b/meson.options @@ -1 +1,7 @@ -option('cy_line_trace', type : 'boolean', value : false) \ No newline at end of file +option('cy_line_trace', type : 'boolean', value : false) + +option('use-ilp64', type: 'boolean', value: false, + description: 'Use ILP64 (64-bit integer) BLAS and LAPACK interfaces') + +option('mkl-threading', type: 'string', value: 'auto', + description: 'MKL threading method, one of: `seq`, `iomp`, `gomp`, `tbb`') \ No newline at end of file diff --git a/pydiso/meson.build b/pydiso/meson.build index 3fe0aaf..d2f13ec 100644 --- a/pydiso/meson.build +++ b/pydiso/meson.build @@ -34,9 +34,39 @@ endif inc_np = include_directories(incdir_numpy) np_dep = declare_dependency(include_directories: inc_np) +#MKL stuffs + +use_ilp64 = get_option('use-ilp64') +if use_ilp64 + blas_interface = ['interface: ilp64'] +else + blas_interface = ['interface: lp64'] +endif + +# MKL-specific options +_threading_opt = get_option('mkl-threading') +if _threading_opt == 'auto' + # Switch default to iomp once conda-forge missing openmp.pc issue is fixed + mkl_opts = ['threading: seq'] +else + mkl_opts = ['threading: ' + _threading_opt] +endif +blas_opts = {'mkl': mkl_opts} +mkl_version_req = '>=2023.0' # see gh-24824 +mkl_may_use_sdl = not use_ilp64 and _threading_opt in ['auto', 'iomp'] # find mkl -mkl_dep = dependency('mkl-sdl', required: true) +mkl_dep = dependency('mkl', + modules: ['cblas'] + blas_interface + mkl_opts, + required: false, # may be required, but we need to emit a custom error message + version: mkl_version_req, +) +# Insert a second try with MKL, because we may be rejecting older versions +# or missing it because no pkg-config installed. If so, we need to retry +# with MKL SDL, and drop the version constraint (this always worked). +if not mkl_dep.found() and mkl_may_use_sdl + mkl_dep = dependency('mkl', modules: ['cblas', 'sdl: true'], required: true) +endif # Deal with M_PI & friends; add `use_math_defines` to c_args or cpp_args # Cython doesn't always get this right itself (see, e.g., gh-16800), so From 63b8c866217d13d151174780686e8ed0bc404d37 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 8 Jan 2024 12:33:31 -0700 Subject: [PATCH 06/53] update github workflow, reenable thread locks --- .github/workflows/python-package-conda.yml | 19 ++++--------------- pydiso/mkl_solver.pyx | 7 ++++--- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index ca841fc..6b86f42 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -23,7 +23,7 @@ jobs: fail-fast: True matrix: os: [ubuntu-latest, macOS-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 @@ -33,28 +33,17 @@ jobs: auto-update-conda: true activate-environment: dev python-version: ${{ matrix.python-version }} + - name: Install Env run: | conda info conda list conda config --show - conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel + conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja - name: Install Our Package - if: ${{ matrix.os != 'windows-latest' }} - run: | - echo $CONDA_PREFIX - export TEST_COV=1 - pip install -v -e . - conda list - - - name: Install Our Package on Windows - if: ${{ matrix.os == 'windows-latest' }} run: | - echo $CONDA_PREFIX - export MKLROOT="$CONDA_PREFIX\Library" - echo $MKLROOT - pip install -v -e . + python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v conda list - name: Run Tests diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 8473802..12b2afd 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -560,16 +560,17 @@ cdef class MKLPardisoSolver: cdef int_t error=0 cdef long_t error64=0, phase64=phase, nrhs64=nrhs - #PyThread_acquire_lock(self.lock, 1) + PyThread_acquire_lock(self.lock, 1) if self._is_32: pardiso(self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, &phase, &self._par.n, self.a, &self._par.ia[0], &self._par.ja[0], &self._par.perm[0], &nrhs, self._par.iparm, &self._par.msglvl, b, x, &error) #PyThread_release_lock(self.lock) - return error else: pardiso_64(self.handle, &self._par64.maxfct, &self._par64.mnum, &self._par64.mtype, &phase64, &self._par64.n, self.a, &self._par64.ia[0], &self._par64.ja[0], &self._par64.perm[0], &nrhs64, self._par64.iparm, &self._par64.msglvl, b, x, &error64) #PyThread_release_lock(self.lock) - return error64 + PyThread_release_lock(self.lock) + error = error or error64 + return error From 2d2de6c36aca7e61b183a896385fd81c553fe2ff Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 8 Jan 2024 13:14:52 -0700 Subject: [PATCH 07/53] update meson build file for mkl linkage parameters --- pydiso/meson.build | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pydiso/meson.build b/pydiso/meson.build index d2f13ec..2657ce2 100644 --- a/pydiso/meson.build +++ b/pydiso/meson.build @@ -34,38 +34,38 @@ endif inc_np = include_directories(incdir_numpy) np_dep = declare_dependency(include_directories: inc_np) -#MKL stuffs +# MKL-specific options +mkl_dep_name = 'mkl-dynamic' use_ilp64 = get_option('use-ilp64') if use_ilp64 - blas_interface = ['interface: ilp64'] + mkl_dep_name += '-ilp64' else - blas_interface = ['interface: lp64'] + mkl_dep_name += '-lp64' endif # MKL-specific options _threading_opt = get_option('mkl-threading') if _threading_opt == 'auto' - # Switch default to iomp once conda-forge missing openmp.pc issue is fixed - mkl_opts = ['threading: seq'] + # openmp.pc not included with conda-forge distribution (yet) + mkl_dep_name += '-seq' else - mkl_opts = ['threading: ' + _threading_opt] + mkl_dep_name += '-' + _threading_opt endif -blas_opts = {'mkl': mkl_opts} -mkl_version_req = '>=2023.0' # see gh-24824 + +mkl_version_req = '>=2023.0' mkl_may_use_sdl = not use_ilp64 and _threading_opt in ['auto', 'iomp'] -# find mkl -mkl_dep = dependency('mkl', - modules: ['cblas'] + blas_interface + mkl_opts, +# find mkl +mkl_dep = dependency(mkl_dep_name, required: false, # may be required, but we need to emit a custom error message version: mkl_version_req, ) # Insert a second try with MKL, because we may be rejecting older versions # or missing it because no pkg-config installed. If so, we need to retry -# with MKL SDL, and drop the version constraint (this always worked). +# with MKL SDL, and drop the version constraint. if not mkl_dep.found() and mkl_may_use_sdl - mkl_dep = dependency('mkl', modules: ['cblas', 'sdl: true'], required: true) + mkl_dep = dependency('mkl-sdl', required: true) endif # Deal with M_PI & friends; add `use_math_defines` to c_args or cpp_args From d558c5644d5f1564c68301639d718359d853d3bd Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 8 Jan 2024 13:15:00 -0700 Subject: [PATCH 08/53] remove print outs --- pydiso/mkl_solver.pyx | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 12b2afd..30e2ea6 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -296,7 +296,6 @@ cdef class MKLPardisoSolver: #set integer length integer_len = A.indices.itemsize self._is_32 = integer_len == sizeof(int_t) - print("here1") if self._is_32: self._par = _PardisoParams() self._initialize(self._par, A, matrix_type, verbose) @@ -306,23 +305,18 @@ cdef class MKLPardisoSolver: else: raise PardisoError("Unrecognized integer length") self._initialized = True - print("here2") if(verbose): #for reporting factorization progress via python's `print` mkl_set_progress(mkl_progress) else: mkl_set_progress(mkl_no_progress) - print("here3") self._set_A(A.data) - print("here4") self._analyze() - print("here5") self._factored = False if factor: self._factor() - print("here6") def refactor(self, A): """solver.refactor(A) @@ -512,7 +506,7 @@ cdef class MKLPardisoSolver: cdef long_t phase64=-1, nrhs64=0, error64=0 if self._initialized: - #PyThread_acquire_lock(self.lock, 1) + PyThread_acquire_lock(self.lock, 1) if self._is_32: pardiso( self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, @@ -525,12 +519,12 @@ cdef class MKLPardisoSolver: &phase64, &self._par64.n, self.a, NULL, NULL, NULL, &nrhs64, self._par64.iparm, &self._par64.msglvl, NULL, NULL, &error64 ) - #PyThread_release_lock(self.lock) + PyThread_release_lock(self.lock) err = error or error64 if err!=0: raise PardisoError("Memmory release error "+_err_messages[err]) #dealloc lock - #PyThread_free_lock(self.lock) + PyThread_free_lock(self.lock) cdef _analyze(self): #phase = 11 @@ -543,8 +537,10 @@ cdef class MKLPardisoSolver: self._factored = False err = self._run_pardiso(22) + if err!=0: raise PardisoError("Factor step error, "+_err_messages[err]) + self._factored = True cdef _solve(self, void* b, void* x, int_t nrhs_in): @@ -556,7 +552,7 @@ cdef class MKLPardisoSolver: if err!=0: raise PardisoError("Solve step error, "+_err_messages[err]) - cdef int _run_pardiso(self, int_t phase, void* b=NULL, void* x=NULL, int_t nrhs=0) nogil: + cdef int _run_pardiso(self, int_t phase, void* b=NULL, void* x=NULL, int_t nrhs=0): cdef int_t error=0 cdef long_t error64=0, phase64=phase, nrhs64=nrhs @@ -565,12 +561,10 @@ cdef class MKLPardisoSolver: pardiso(self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, &phase, &self._par.n, self.a, &self._par.ia[0], &self._par.ja[0], &self._par.perm[0], &nrhs, self._par.iparm, &self._par.msglvl, b, x, &error) - #PyThread_release_lock(self.lock) else: pardiso_64(self.handle, &self._par64.maxfct, &self._par64.mnum, &self._par64.mtype, &phase64, &self._par64.n, self.a, &self._par64.ia[0], &self._par64.ja[0], &self._par64.perm[0], &nrhs64, self._par64.iparm, &self._par64.msglvl, b, x, &error64) - #PyThread_release_lock(self.lock) PyThread_release_lock(self.lock) error = error or error64 return error From c430f1ffeaf4b51f681a6e858859301ed09893e3 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 8 Jan 2024 13:16:31 -0700 Subject: [PATCH 09/53] remove setup.py file --- pydiso/setup.py | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 pydiso/setup.py diff --git a/pydiso/setup.py b/pydiso/setup.py deleted file mode 100644 index f951490..0000000 --- a/pydiso/setup.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -from os.path import join, abspath, dirname - -base_path = abspath(dirname(__file__)) - - -# Enable line tracing for coverage of cython files conditionally -ext_kwargs = {} -if os.environ.get("TEST_COV", None) is not None: - ext_kwargs["define_macros"] = [("CYTHON_TRACE_NOGIL", 1)] - - -def configuration(parent_package="", top_path=None): - from numpy.distutils.misc_util import Configuration, get_numpy_include_dirs - import numpy.distutils.system_info as sysinfo - config = Configuration("pydiso", parent_package, top_path) - - try: - from Cython.Build import cythonize - cythonize(join(base_path, "mkl_solver.pyx")) - except ImportError: - pass - - # get information about mkl location - mkl_root = os.environ.get('MKLROOT', None) - if mkl_root is None: - mkl_info = sysinfo.get_info('mkl') - else: - mkl_info = { - 'include_dirs': [join(mkl_root, 'include')], - 'library_dirs': [join(mkl_root, 'lib'), join(mkl_root, 'lib', 'intel64')], - 'libraries': ['mkl_rt'] - } - - mkl_include_dirs = mkl_info.get('include_dirs', []) - mkl_library_dirs = mkl_info.get('library_dirs', []) - mkl_libraries = mkl_info.get('libraries', ['mkl_rt']) - - config.add_extension( - "mkl_solver", - sources=["mkl_solver.c"], - libraries=mkl_libraries, - include_dirs=get_numpy_include_dirs() + mkl_include_dirs, - library_dirs=mkl_library_dirs, - extra_compile_args=['-w'], - **ext_kwargs - ) - - return config From f462fdeb018b4b479f14b50ef31fc123d508b872 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 8 Jan 2024 13:48:05 -0700 Subject: [PATCH 10/53] install setuptools --- .github/workflows/python-package-conda.yml | 2 +- pyproject.toml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 6b86f42..ed8685c 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -39,7 +39,7 @@ jobs: conda info conda list conda config --show - conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja + conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm - name: Install Our Package run: | diff --git a/pyproject.toml b/pyproject.toml index 1645e8c..e93252c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,8 +9,6 @@ requires = [ # This package automatically provides all of the numpy pinning for different python # versions and runtime requirements. "oldest-supported-numpy", - # oldest mkl-devel available (for building against). - "mkl-devel==2023" # The following is taken from scipy's pyproject.toml file to handle # building against the proper numpy API From 368508a8beaef88e773171cd38a9acbcb80750e9 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 8 Jan 2024 13:53:48 -0700 Subject: [PATCH 11/53] add option to force sdl usage --- meson.options | 3 +++ pydiso/meson.build | 29 +++++++++++++++++++---------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/meson.options b/meson.options index 1ae0e79..6f73493 100644 --- a/meson.options +++ b/meson.options @@ -1,5 +1,8 @@ option('cy_line_trace', type : 'boolean', value : false) +option('use-sdl', type: 'boolean', value: true, + description: 'Use the single dynamic library.') + option('use-ilp64', type: 'boolean', value: false, description: 'Use ILP64 (64-bit integer) BLAS and LAPACK interfaces') diff --git a/pydiso/meson.build b/pydiso/meson.build index 2657ce2..0b4d738 100644 --- a/pydiso/meson.build +++ b/pydiso/meson.build @@ -54,18 +54,27 @@ else endif mkl_version_req = '>=2023.0' -mkl_may_use_sdl = not use_ilp64 and _threading_opt in ['auto', 'iomp'] -# find mkl -mkl_dep = dependency(mkl_dep_name, - required: false, # may be required, but we need to emit a custom error message - version: mkl_version_req, -) -# Insert a second try with MKL, because we may be rejecting older versions -# or missing it because no pkg-config installed. If so, we need to retry -# with MKL SDL, and drop the version constraint. -if not mkl_dep.found() and mkl_may_use_sdl +use_sdl = get_option('use-sdl') + +if use_sdl mkl_dep = dependency('mkl-sdl', required: true) +else + # find mkl + mkl_dep = dependency(mkl_dep_name, + required: false, # may be required, but we need to emit a custom error message + version: mkl_version_req, + ) + + mkl_may_use_sdl = not use_ilp64 and _threading_opt in ['auto', 'iomp'] + + # Insert a second try with MKL, because we may be rejecting older versions + # or missing it because no pkg-config installed. If so, we need to retry + # with MKL SDL, and drop the version constraint. + if not mkl_dep.found() and mkl_may_use_sdl + mkl_dep = dependency('mkl-sdl', required: true) + endif + endif # Deal with M_PI & friends; add `use_math_defines` to c_args or cpp_args From d635c8a0fee43bae4101f1cd0ec6cfa87db2c573 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 28 Feb 2024 12:55:53 -0700 Subject: [PATCH 12/53] do not fail fast on tests --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index ed8685c..b6c0cb7 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -20,7 +20,7 @@ jobs: run: shell: bash -l {0} strategy: - fail-fast: True + fail-fast: False matrix: os: [ubuntu-latest, macOS-latest, windows-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] From d45857bc00557518f95b0212e6b94819a466cf2e Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 28 Feb 2024 15:23:33 -0700 Subject: [PATCH 13/53] move thread lock allocation --- pydiso/mkl_solver.pyx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 30e2ea6..e0aa967 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -1,6 +1,7 @@ #cython: language_level=3 #cython: linetrace=True cimport numpy as np +import cython from cython cimport numeric from cpython.pythread cimport ( PyThread_type_lock, @@ -260,9 +261,6 @@ cdef class MKLPardisoSolver: raise ValueError("Matrix is not square") self.shape = n_row, n_col - # allocate the lock - #self.lock = PyThread_allocate_lock() - self._data_type = A.dtype if matrix_type is None: if np.issubdtype(self._data_type, np.complexfloating): @@ -306,6 +304,9 @@ cdef class MKLPardisoSolver: raise PardisoError("Unrecognized integer length") self._initialized = True + # allocate the lock + self.lock = PyThread_allocate_lock() + if(verbose): #for reporting factorization progress via python's `print` mkl_set_progress(mkl_progress) @@ -523,8 +524,8 @@ cdef class MKLPardisoSolver: err = error or error64 if err!=0: raise PardisoError("Memmory release error "+_err_messages[err]) - #dealloc lock - PyThread_free_lock(self.lock) + #dealloc lock + PyThread_free_lock(self.lock) cdef _analyze(self): #phase = 11 @@ -552,7 +553,8 @@ cdef class MKLPardisoSolver: if err!=0: raise PardisoError("Solve step error, "+_err_messages[err]) - cdef int _run_pardiso(self, int_t phase, void* b=NULL, void* x=NULL, int_t nrhs=0): + @cython.boundscheck(False) + cdef int _run_pardiso(self, int_t phase, void* b=NULL, void* x=NULL, int_t nrhs=0) nogil: cdef int_t error=0 cdef long_t error64=0, phase64=phase, nrhs64=nrhs From 4c9a9d721d75924bf21be4fd7099a44c16ce4486 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 28 Feb 2024 17:07:14 -0700 Subject: [PATCH 14/53] update some meta information --- .bumpversion.cfg | 3 --- .github/workflows/python-package-conda.yml | 4 ++-- README.md | 27 ++++++++-------------- pydiso/__init__.py | 17 +++++++++++++- 4 files changed, 28 insertions(+), 23 deletions(-) delete mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index b3bbe38..0000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[bumpversion] -current_version = 0.0.3 -files = setup.py pydiso/__init__.py diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index b6c0cb7..2dd4f0d 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -79,7 +79,7 @@ jobs: conda info conda list conda config --show - conda install --quiet --yes pip numpy scipy cython mkl mkl-devel + conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm - name: Install Our Package run: | @@ -88,7 +88,7 @@ jobs: - name: Generate Source Distribution run: | - python setup.py sdist + python -m build --no-isolation --skip-dependency-check --sdist . - name: pypi-publish uses: pypa/gh-action-pypi-publish@v1.4.2 diff --git a/README.md b/README.md index 814e90c..f003fee 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,29 @@ # pydiso Pydiso is a python wrapper for the pardiso solver. It is currently implemented for the -Intel MKL's version of pardiso. It's goal is to expose the full functionality of pardiso +Intel MKL's version of pardiso. Its goal is to expose the full functionality of pardiso to python, allowing the user to customize it as necessary for their use cases. # Installation -### Installing from source +## Installing from source The wrapper is written in cython and links to the mkl libraries dynamically. Therefore, it needs to find the necessary header files associated with the MKL installation to compile. -For conda users, these headers can be installed with `mkl-devel` package that is available -on the default channel, conda-forge channel, the intel channel, or others, e.g. +The meson build backend uses pkg-config to identify the locations of the mkl header files +and library dynamic libraries. Most development installations of MKL should provide the +necessary pkg-config files for this. For example, conda users can be install the necessary +configuration information with `mkl-devel` package that is available on the default channel, +conda-forge channel, the intel channel, or others, e.g. `conda install mkl-devel` -Most of the time, your respective numpy installations will then be intelligent enough to -identify the location of the installed MKL. However if you run into issues automatically -finding the library headers, you will need to set the `MKLROOT` environment variable to -point to the correct location. On Linux and MacOS the `mkl-rt` library and `mkl.h` are -usually in the same folder, and thus the root should point to that directory. On Windows -the `mkl-rt.h` and `mkl-rt.lib` are not in the same folder but seperated a level into -`Library` and `Include` directories, and in this case `MKLROOT` would point to the folder -containing them. +If you have installed the configuration files to a non-standard location, you will need to set +`PKG_CONFIG_PATH` to point to that location. -After the necessary MKL files are accessible, you should be able to do the standard install -script common to python packages by running either -`python setup.py install` - -or, equivalently +After the necessary MKL files are accessible, you should be able to install by running `pip install .` diff --git a/pydiso/__init__.py b/pydiso/__init__.py index 759868c..3ca6138 100644 --- a/pydiso/__init__.py +++ b/pydiso/__init__.py @@ -1,4 +1,19 @@ -__version__ = "0.0.3" __author__ = "SimPEG Team" __license__ = "MIT" __copyright__ = "2021, SimPEG Developers, http://simpeg.xyz" + +from importlib.metadata import version, PackageNotFoundError + +# Version +try: + # - Released versions just tags: 0.8.0 + # - GitHub commits add .dev#+hash: 0.8.1.dev4+g2785721 + # - Uncommitted changes add timestamp: 0.8.1.dev4+g2785721.d20191022 + __version__ = version("discretize") +except PackageNotFoundError: + # If it was not installed, then we don't know the version. We could throw a + # warning here, but this case *should* be rare. discretize should be + # installed properly! + from datetime import datetime + + __version__ = "unknown-" + datetime.today().strftime("%Y%m%d") From 6d11442becbbaf875184d9508089563dbda6f560 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Thu, 29 Feb 2024 10:39:31 -0700 Subject: [PATCH 15/53] ignore pycharm files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f107869..4100ca3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ __pycache__ mkl_solver.c .coverage coverage.xml + +.idea/ From 4ed000d620ccecc8e40d2d8eb6ac2468c81997aa Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Thu, 29 Feb 2024 10:40:03 -0700 Subject: [PATCH 16/53] be a bit more accurate about the argument types --- pydiso/mkl_solver.pyx | 109 ++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 57 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index e0aa967..163607f 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -16,14 +16,11 @@ import numpy as np import scipy.sparse as sp import os -ctypedef long long MKL_INT64 -ctypedef unsigned long long MKL_UINT64 -ctypedef int MKL_INT - -ctypedef MKL_INT int_t -ctypedef MKL_INT64 long_t - cdef extern from 'mkl.h': + + ctypedef long long MKL_INT64 + ctypedef unsigned long long MKL_UINT64 + ctypedef int MKL_INT int MKL_DOMAIN_PARDISO ctypedef struct MKLVersion: @@ -42,28 +39,27 @@ cdef extern from 'mkl.h': int mkl_get_max_threads() int mkl_domain_get_max_threads(int domain) - ctypedef int (*ProgressEntry)(int_t* thread, int_t* step, char* stage, int_t stage_len) except? -1; + ctypedef int (*ProgressEntry)(int* thread, int* step, char* stage, int stage_len) except? -1; ProgressEntry mkl_set_progress(ProgressEntry progress); ctypedef void * _MKL_DSS_HANDLE_t - void pardiso(_MKL_DSS_HANDLE_t, const int*, const int*, const int*, - const int *, const int *, const void *, const int *, - const int *, int *, const int *, int *, - const int *, void *, void *, int *) nogil - - void pardiso_64(_MKL_DSS_HANDLE_t, const long_t *, const long_t *, const long_t *, - const long_t *, const long_t *, const void *, const long_t *, - const long_t *, long_t *, const long_t *, long_t *, - const long_t *, void *, void *, long_t *) nogil + void pardiso(_MKL_DSS_HANDLE_t, const MKL_INT*, const MKL_INT*, const MKL_INT*, + const MKL_INT *, const MKL_INT *, const void *, const MKL_INT *, + const MKL_INT *, MKL_INT *, const MKL_INT *, MKL_INT *, + const MKL_INT *, void *, void *, MKL_INT *) nogil + void pardiso_64(_MKL_DSS_HANDLE_t, const long long int *, const long long int *, const long long int *, + const long long int *, const long long int *, const void *, const long long int *, + const long long int *, long long int *, const long long int *, long long int *, + const long long int *, void *, void *, long long int *) nogil #call pardiso (pt, maxfct, mnum, mtype, phase, n, a, ia, ja, perm, nrhs, iparm, msglvl, b, x, error) -cdef int mkl_progress(int_t *thread, int_t* step, char* stage, int_t stage_len): +cdef int mkl_progress(int *thread, int* step, char* stage, int stage_len): print(thread[0], step[0], stage, stage_len) return 0 -cdef int mkl_no_progress(int_t *thread, int_t* step, char* stage, int_t stage_len) nogil: +cdef int mkl_no_progress(int *thread, int* step, char* stage, int stage_len) nogil: return 0 MATRIX_TYPES ={ @@ -170,14 +166,14 @@ def get_mkl_version(): return vers cdef class _PardisoParams: - cdef int_t iparm[64] - cdef int_t n, mtype, maxfct, mnum, msglvl - cdef int_t[:] ia, ja, perm + cdef MKL_INT iparm[64] + cdef MKL_INT n, mtype, maxfct, mnum, msglvl + cdef MKL_INT[:] ia, ja, perm cdef class _PardisoParams64: - cdef long_t iparm[64] - cdef long_t n, mtype, maxfct, mnum, msglvl - cdef long_t[:] ia, ja, perm + cdef MKL_INT64 iparm[64] + cdef MKL_INT64 n, mtype, maxfct, mnum, msglvl + cdef MKL_INT64[:] ia, ja, perm ctypedef fused _par_params: _PardisoParams @@ -187,11 +183,11 @@ cdef class MKLPardisoSolver: cdef _MKL_DSS_HANDLE_t handle[64] cdef _PardisoParams _par cdef _PardisoParams64 _par64 - cdef int_t _is_32 - cdef int_t mat_type - cdef int_t _factored + cdef int _call32 + cdef int mat_type + cdef int _factored cdef size_t shape[2] - cdef int_t _initialized + cdef int _initialized cdef PyThread_type_lock lock cdef void * a @@ -293,8 +289,10 @@ cdef class MKLPardisoSolver: #set integer length integer_len = A.indices.itemsize - self._is_32 = integer_len == sizeof(int_t) - if self._is_32: + # we only need to call the 64 bit version if + # sizeof(MKL_INT) == 4 and A.indices.itemsize == 8 + self._call32 = not (sizeof(MKL_INT) == 4 and integer_len == 8) + if self._call32: self._par = _PardisoParams() self._initialize(self._par, A, matrix_type, verbose) elif integer_len == 8: @@ -307,7 +305,7 @@ cdef class MKLPardisoSolver: # allocate the lock self.lock = PyThread_allocate_lock() - if(verbose): + if verbose: #for reporting factorization progress via python's `print` mkl_set_progress(mkl_progress) else: @@ -399,7 +397,7 @@ cdef class MKLPardisoSolver: if bp == xp: raise PardisoError("b and x must be different arrays") - cdef int_t nrhs = b.shape[1] if b.ndim == 2 else 1 + cdef MKL_INT nrhs = b.shape[1] if b.ndim == 2 else 1 if transpose: self.set_iparm(11, 2) @@ -412,7 +410,7 @@ cdef class MKLPardisoSolver: def perm(self): """ Fill-reducing permutation vector used inside pardiso. """ - if self._is_32: + if self._call32: return np.array(self._par.perm) else: return np.array(self._par64.perm) @@ -421,12 +419,12 @@ cdef class MKLPardisoSolver: def iparm(self): """ Parameter options for the pardiso solver. """ - if self._is_32: + if self._call32: return np.array(self._par.iparm) else: return np.array(self._par64.iparm) - def set_iparm(self, int_t i, int_t val): + def set_iparm(self, MKL_INT i, MKL_INT val): if i > 63 or i < 0: raise IndexError(f"index {i} is out of bounds for size 64 array") if i not in [ @@ -434,7 +432,7 @@ cdef class MKLPardisoSolver: 24, 26, 30, 33, 34, 35, 36, 38, 42, 55, 59 ]: raise PardisoError(f"cannot set parameter {i} of the iparm array") - if self._is_32: + if self._call32: self._par.iparm[i] = val else: self._par64.iparm[i] = val @@ -444,8 +442,12 @@ cdef class MKLPardisoSolver: return self.iparm[17] cdef _initialize(self, _par_params par, A, matrix_type, verbose): + if sizeof(MKL_INT) == 4: + np_int_dtype = np.int32 + else: + np_int_dtype = np.int64 par.n = A.shape[0] - par.perm = np.empty(par.n, dtype=np.int32) + par.perm = np.empty(par.n, dtype=np_int_dtype) par.maxfct = 1 par.mnum = 1 @@ -487,28 +489,21 @@ cdef class MKLPardisoSolver: par.iparm[55] = 0 # Internal function used to work with pivot and calculation of diagonal arrays turned off. par.iparm[59] = 0 # operate in-core mode - if _par_params is _PardisoParams: - indices = np.require(A.indices, dtype=np.int32) - indptr = np.require(A.indptr, dtype=np.int32) - else: - indices = np.require(A.indices, dtype=np.int64) - indptr = np.require(A.indptr, dtype=np.int64) - - par.ia = indptr - par.ja = indices + par.ia = np.require(A.indices, dtype=np_int_dtype) + par.ja = np.require(A.indptr, dtype=np_int_dtype) cdef _set_A(self, data): - self._Adata = data + self._Adata = np.ascontiguousarray(data) self.a = np.PyArray_DATA(data) def __dealloc__(self): # Need to call pardiso with phase=-1 to release memory - cdef int_t phase=-1, nrhs=0, error=0 - cdef long_t phase64=-1, nrhs64=0, error64=0 + cdef MKL_INT phase=-1, nrhs=0, error=0 + cdef MKL_INT64 phase64=-1, nrhs64=0, error64=0 if self._initialized: PyThread_acquire_lock(self.lock, 1) - if self._is_32: + if self._call32: pardiso( self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, &phase, &self._par.n, self.a, NULL, NULL, NULL, &nrhs, self._par.iparm, @@ -544,7 +539,7 @@ cdef class MKLPardisoSolver: self._factored = True - cdef _solve(self, void* b, void* x, int_t nrhs_in): + cdef _solve(self, void* b, void* x, MKL_INT nrhs_in): #phase = 33 if(not self._factored): raise PardisoError("Cannot solve without a previous factorization.") @@ -554,12 +549,12 @@ cdef class MKLPardisoSolver: raise PardisoError("Solve step error, "+_err_messages[err]) @cython.boundscheck(False) - cdef int _run_pardiso(self, int_t phase, void* b=NULL, void* x=NULL, int_t nrhs=0) nogil: - cdef int_t error=0 - cdef long_t error64=0, phase64=phase, nrhs64=nrhs + cdef MKL_INT _run_pardiso(self, MKL_INT phase, void* b=NULL, void* x=NULL, MKL_INT nrhs=0) nogil: + cdef MKL_INT error=0 + cdef MKL_INT64 error64=0, phase64=phase, nrhs64=nrhs PyThread_acquire_lock(self.lock, 1) - if self._is_32: + if self._call32: pardiso(self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, &phase, &self._par.n, self.a, &self._par.ia[0], &self._par.ja[0], &self._par.perm[0], &nrhs, self._par.iparm, &self._par.msglvl, b, x, &error) @@ -568,5 +563,5 @@ cdef class MKLPardisoSolver: &phase64, &self._par64.n, self.a, &self._par64.ia[0], &self._par64.ja[0], &self._par64.perm[0], &nrhs64, self._par64.iparm, &self._par64.msglvl, b, x, &error64) PyThread_release_lock(self.lock) - error = error or error64 + error = error or error64 return error From d762962e9b2a05cbf4df1e5ed9d0866ecd417230 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Thu, 29 Feb 2024 14:43:02 -0700 Subject: [PATCH 17/53] Be a little safer about free the lock --- pydiso/mkl_solver.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 163607f..6634eb2 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -520,7 +520,9 @@ cdef class MKLPardisoSolver: if err!=0: raise PardisoError("Memmory release error "+_err_messages[err]) #dealloc lock + if self.lock is not NULL: PyThread_free_lock(self.lock) + self.lock = NULL cdef _analyze(self): #phase = 11 From 5fa9adeb4eb5e699450fd5d98fb804d666b95308 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Thu, 29 Feb 2024 15:00:35 -0700 Subject: [PATCH 18/53] split __dealloc__ into a dealloc and a __del__ method for safety --- pydiso/mkl_solver.pyx | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 6634eb2..b336857 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -194,6 +194,10 @@ cdef class MKLPardisoSolver: cdef object _data_type cdef object _Adata #a reference to make sure the pointer "a" doesn't get destroyed + def __cinit__(self, *args, **kwargs): + # allocate the lock + self.lock = PyThread_allocate_lock() + def __init__(self, A, matrix_type=None, factor=True, verbose=False): '''ParidsoSolver(A, matrix_type=None, factor=True, verbose=False) An interface to the intel MKL pardiso sparse matrix solver. @@ -302,9 +306,6 @@ cdef class MKLPardisoSolver: raise PardisoError("Unrecognized integer length") self._initialized = True - # allocate the lock - self.lock = PyThread_allocate_lock() - if verbose: #for reporting factorization progress via python's `print` mkl_set_progress(mkl_progress) @@ -496,7 +497,7 @@ cdef class MKLPardisoSolver: self._Adata = np.ascontiguousarray(data) self.a = np.PyArray_DATA(data) - def __dealloc__(self): + def __del__(self): # Need to call pardiso with phase=-1 to release memory cdef MKL_INT phase=-1, nrhs=0, error=0 cdef MKL_INT64 phase64=-1, nrhs64=0, error64=0 @@ -519,10 +520,12 @@ cdef class MKLPardisoSolver: err = error or error64 if err!=0: raise PardisoError("Memmory release error "+_err_messages[err]) - #dealloc lock - if self.lock is not NULL: - PyThread_free_lock(self.lock) - self.lock = NULL + self._initialized = False + + + def __dealloc__(self): + #dealloc lock + PyThread_free_lock(self.lock) cdef _analyze(self): #phase = 11 From bdd2da7b93bbdd7e3e9f87bea6603886f97168ce Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Thu, 29 Feb 2024 15:19:23 -0700 Subject: [PATCH 19/53] determine initialization status dynamically --- pydiso/mkl_solver.pyx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index b336857..40e7568 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -187,7 +187,6 @@ cdef class MKLPardisoSolver: cdef int mat_type cdef int _factored cdef size_t shape[2] - cdef int _initialized cdef PyThread_type_lock lock cdef void * a @@ -255,7 +254,6 @@ cdef class MKLPardisoSolver: >>> np.allclose(x, x_solved) True ''' - self._initialized = False n_row, n_col = A.shape if n_row != n_col: raise ValueError("Matrix is not square") @@ -304,7 +302,6 @@ cdef class MKLPardisoSolver: self._initialize(self._par64, A, matrix_type, verbose) else: raise PardisoError("Unrecognized integer length") - self._initialized = True if verbose: #for reporting factorization progress via python's `print` @@ -346,6 +343,15 @@ cdef class MKLPardisoSolver: def __call__(self, b): return self.solve(b) + @property + cpdef int _initialized(self): + # this says if the handle was ever initialized + cdef int i + for i in range(64): + if self.handle[i] != 0: + return 1 + return 0 + def solve(self, b, x=None, transpose=False): """solve(self, b, x=None, transpose=False) Solves the equation AX=B using the factored A matrix From e52dc867979a3108cac3b0a1eab69ec5a2d05482 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Thu, 29 Feb 2024 15:25:00 -0700 Subject: [PATCH 20/53] make _initialized a function --- pydiso/mkl_solver.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 40e7568..fb1af22 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -343,7 +343,6 @@ cdef class MKLPardisoSolver: def __call__(self, b): return self.solve(b) - @property cpdef int _initialized(self): # this says if the handle was ever initialized cdef int i @@ -508,7 +507,8 @@ cdef class MKLPardisoSolver: cdef MKL_INT phase=-1, nrhs=0, error=0 cdef MKL_INT64 phase64=-1, nrhs64=0, error64=0 - if self._initialized: + # Only need to deallocate if the handle itself was ever allocated. + if self._initialized(): PyThread_acquire_lock(self.lock, 1) if self._call32: pardiso( @@ -526,7 +526,6 @@ cdef class MKLPardisoSolver: err = error or error64 if err!=0: raise PardisoError("Memmory release error "+_err_messages[err]) - self._initialized = False def __dealloc__(self): @@ -576,3 +575,4 @@ cdef class MKLPardisoSolver: PyThread_release_lock(self.lock) error = error or error64 return error + From 8b967a60275057010973bbc81616f1961331235d Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Thu, 29 Feb 2024 15:31:12 -0700 Subject: [PATCH 21/53] just use if self.handle[i] --- pydiso/mkl_solver.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index fb1af22..8177759 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -347,7 +347,7 @@ cdef class MKLPardisoSolver: # this says if the handle was ever initialized cdef int i for i in range(64): - if self.handle[i] != 0: + if self.handle[i]: return 1 return 0 From bbcdbd5d774556f3f436256482e60a9a3a22333a Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Thu, 29 Feb 2024 15:36:41 -0700 Subject: [PATCH 22/53] use mamba --- .github/workflows/python-package-conda.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 2dd4f0d..0d39ea2 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -26,20 +26,22 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: - auto-update-conda: true - activate-environment: dev python-version: ${{ matrix.python-version }} + mamba-version: '*' + channels: conda-forge, defaults + channel-priority: true + activate-environment: dev - name: Install Env run: | conda info conda list conda config --show - conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm + mamba install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm - name: Install Our Package run: | From a567a732954f67d55bf5120eac9751e6a32a75b3 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Thu, 29 Feb 2024 15:53:13 -0700 Subject: [PATCH 23/53] cache the environment --- .github/workflows/python-package-conda.yml | 32 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 0d39ea2..5309b45 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -12,6 +12,10 @@ on: schedule: - cron: "0 13 * * 1" +env: + CACHE_NUMBER: 0 # increase to reset cache manually + + jobs: build-and-test: name: Testing (${{ matrix.python-version }}, ${{ matrix.os }}) @@ -22,8 +26,18 @@ jobs: strategy: fail-fast: False matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + include: + - os: ubuntu-latest + label: linux-64 + prefix: /usr/share/miniconda3/envs/my-env + - os: macos-latest + label: osx-64 + prefix: /usr/share/miniconda3/envs/my-env + - os: windows-latest + label: win-64 + prefix: C:\Miniconda3\envs\my-env steps: - uses: actions/checkout@v4 @@ -36,12 +50,26 @@ jobs: channel-priority: true activate-environment: dev - - name: Install Env + - name: Get Date + id: get-date + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + + - uses: actions/cache@v2 + with: + path: ${{ matrix.prefix }} + key: ${{ matrix.label }}-${{ matrix.python-version }}-conda-${{ env.DATE }}-${{ env.CACHE_NUMBER }} + id: cache + + - name: Conda information run: | conda info conda list conda config --show + + - name: Create environment + run: | mamba install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm + if: steps.cache.outputs.cache-hit != 'true' - name: Install Our Package run: | From b233eeda7bd3b2b83d466549f3da36c6bceb0835 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 14:21:26 -0700 Subject: [PATCH 24/53] fix indices and indptr --- pydiso/mkl_solver.pyx | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 8177759..f2f7865 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -54,6 +54,12 @@ cdef extern from 'mkl.h': const long long int *, long long int *, const long long int *, long long int *, const long long int *, void *, void *, long long int *) nogil +if sizeof(MKL_INT) == 4: + _np_mkl_int = np.int32 +else: + _np_mkl_int = np.int64 +_np_mkl_int64 = np.int64 + #call pardiso (pt, maxfct, mnum, mtype, phase, n, a, ia, ja, perm, nrhs, iparm, msglvl, b, x, error) cdef int mkl_progress(int *thread, int* step, char* stage, int stage_len): print(thread[0], step[0], stage, stage_len) @@ -197,6 +203,10 @@ cdef class MKLPardisoSolver: # allocate the lock self.lock = PyThread_allocate_lock() + # handle should be intialized just fine, but just in case.. + for i in range(64): + self.handle[64] = NULL + def __init__(self, A, matrix_type=None, factor=True, verbose=False): '''ParidsoSolver(A, matrix_type=None, factor=True, verbose=False) An interface to the intel MKL pardiso sparse matrix solver. @@ -448,6 +458,10 @@ cdef class MKLPardisoSolver: return self.iparm[17] cdef _initialize(self, _par_params par, A, matrix_type, verbose): + if _par_params is _PardisoParams: + np_int_dtype = _np_mkl_int + else: + np_int_dtype = _np_mkl_int64 if sizeof(MKL_INT) == 4: np_int_dtype = np.int32 else: @@ -495,11 +509,11 @@ cdef class MKLPardisoSolver: par.iparm[55] = 0 # Internal function used to work with pivot and calculation of diagonal arrays turned off. par.iparm[59] = 0 # operate in-core mode - par.ia = np.require(A.indices, dtype=np_int_dtype) - par.ja = np.require(A.indptr, dtype=np_int_dtype) + par.ia = np.require(A.indptr, dtype=np_int_dtype) + par.ja = np.require(A.indices, dtype=np_int_dtype) cdef _set_A(self, data): - self._Adata = np.ascontiguousarray(data) + self._Adata = data self.a = np.PyArray_DATA(data) def __del__(self): @@ -529,8 +543,10 @@ cdef class MKLPardisoSolver: def __dealloc__(self): - #dealloc lock - PyThread_free_lock(self.lock) + if self.lock: + #dealloc lock + PyThread_free_lock(self.lock) + self.lock = NULL cdef _analyze(self): #phase = 11 From d018d0b63315ef3d0834681252be0549ee768dda Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 14:38:42 -0700 Subject: [PATCH 25/53] stop overwriting dtype --- pydiso/__init__.py | 2 +- pydiso/mkl_solver.pyx | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pydiso/__init__.py b/pydiso/__init__.py index 3ca6138..4e72450 100644 --- a/pydiso/__init__.py +++ b/pydiso/__init__.py @@ -9,7 +9,7 @@ # - Released versions just tags: 0.8.0 # - GitHub commits add .dev#+hash: 0.8.1.dev4+g2785721 # - Uncommitted changes add timestamp: 0.8.1.dev4+g2785721.d20191022 - __version__ = version("discretize") + __version__ = version("pydiso") except PackageNotFoundError: # If it was not installed, then we don't know the version. We could throw a # warning here, but this case *should* be rare. discretize should be diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index f2f7865..5787ca4 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -462,10 +462,6 @@ cdef class MKLPardisoSolver: np_int_dtype = _np_mkl_int else: np_int_dtype = _np_mkl_int64 - if sizeof(MKL_INT) == 4: - np_int_dtype = np.int32 - else: - np_int_dtype = np.int64 par.n = A.shape[0] par.perm = np.empty(par.n, dtype=np_int_dtype) @@ -490,7 +486,7 @@ cdef class MKLPardisoSolver: par.iparm[18] = 0 # do not report flop count par.iparm[20] = 1 if matrix_type in [-2, -4, 6] else 0 par.iparm[23] = 0 # classic (not parallel) factorization - par.iparm[24] = 0 # default behavoir of parallel solving + par.iparm[24] = 0 # default behavior of parallel solving par.iparm[26] = 0 # Do not check the input matrix #set precision if self._data_type==np.float64 or self._data_type==np.complex128: From 88f7ea8095d98c805923d6c34417c70e020f1a21 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 15:06:03 -0700 Subject: [PATCH 26/53] undo some types --- pydiso/mkl_solver.pyx | 61 +++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 5787ca4..b0bf486 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -16,11 +16,14 @@ import numpy as np import scipy.sparse as sp import os -cdef extern from 'mkl.h': +ctypedef long long MKL_INT64 +ctypedef unsigned long long MKL_UINT64 +ctypedef int MKL_INT + +ctypedef MKL_INT int_t +ctypedef MKL_INT64 long_t - ctypedef long long MKL_INT64 - ctypedef unsigned long long MKL_UINT64 - ctypedef int MKL_INT +cdef extern from 'mkl.h': int MKL_DOMAIN_PARDISO ctypedef struct MKLVersion: @@ -44,17 +47,17 @@ cdef extern from 'mkl.h': ctypedef void * _MKL_DSS_HANDLE_t - void pardiso(_MKL_DSS_HANDLE_t, const MKL_INT*, const MKL_INT*, const MKL_INT*, - const MKL_INT *, const MKL_INT *, const void *, const MKL_INT *, - const MKL_INT *, MKL_INT *, const MKL_INT *, MKL_INT *, - const MKL_INT *, void *, void *, MKL_INT *) nogil + void pardiso(_MKL_DSS_HANDLE_t, const int*, const int*, const int*, + const int *, const int *, const void *, const int *, + const int *, int *, const int_t *, int *, + const int *, void *, void *, int *) nogil - void pardiso_64(_MKL_DSS_HANDLE_t, const long long int *, const long long int *, const long long int *, - const long long int *, const long long int *, const void *, const long long int *, - const long long int *, long long int *, const long long int *, long long int *, - const long long int *, void *, void *, long long int *) nogil + void pardiso_64(_MKL_DSS_HANDLE_t, const long_t *, const long_t *, const long_t *, + const long_t *, const long_t *, const void *, const long_t *, + const long_t *, long_t *, const long_t *, long_t *, + const long_t *, void *, void *, long_t *) nogil -if sizeof(MKL_INT) == 4: +if sizeof(int_t) == 4: _np_mkl_int = np.int32 else: _np_mkl_int = np.int64 @@ -172,14 +175,14 @@ def get_mkl_version(): return vers cdef class _PardisoParams: - cdef MKL_INT iparm[64] - cdef MKL_INT n, mtype, maxfct, mnum, msglvl - cdef MKL_INT[:] ia, ja, perm + cdef int_t iparm[64] + cdef int_t n, mtype, maxfct, mnum, msglvl + cdef int_t[:] ia, ja, perm cdef class _PardisoParams64: - cdef MKL_INT64 iparm[64] - cdef MKL_INT64 n, mtype, maxfct, mnum, msglvl - cdef MKL_INT64[:] ia, ja, perm + cdef long_t iparm[64] + cdef long_t n, mtype, maxfct, mnum, msglvl + cdef long_t[:] ia, ja, perm ctypedef fused _par_params: _PardisoParams @@ -303,7 +306,7 @@ cdef class MKLPardisoSolver: integer_len = A.indices.itemsize # we only need to call the 64 bit version if # sizeof(MKL_INT) == 4 and A.indices.itemsize == 8 - self._call32 = not (sizeof(MKL_INT) == 4 and integer_len == 8) + self._call32 = not (sizeof(int_t) == 4 and integer_len == 8) if self._call32: self._par = _PardisoParams() self._initialize(self._par, A, matrix_type, verbose) @@ -413,7 +416,7 @@ cdef class MKLPardisoSolver: if bp == xp: raise PardisoError("b and x must be different arrays") - cdef MKL_INT nrhs = b.shape[1] if b.ndim == 2 else 1 + cdef int_t nrhs = b.shape[1] if b.ndim == 2 else 1 if transpose: self.set_iparm(11, 2) @@ -440,7 +443,7 @@ cdef class MKLPardisoSolver: else: return np.array(self._par64.iparm) - def set_iparm(self, MKL_INT i, MKL_INT val): + def set_iparm(self, int i, int val): if i > 63 or i < 0: raise IndexError(f"index {i} is out of bounds for size 64 array") if i not in [ @@ -514,8 +517,8 @@ cdef class MKLPardisoSolver: def __del__(self): # Need to call pardiso with phase=-1 to release memory - cdef MKL_INT phase=-1, nrhs=0, error=0 - cdef MKL_INT64 phase64=-1, nrhs64=0, error64=0 + cdef int_t phase=-1, nrhs=0, error=0 + cdef long_t phase64=-1, nrhs64=0, error64=0 # Only need to deallocate if the handle itself was ever allocated. if self._initialized(): @@ -561,7 +564,7 @@ cdef class MKLPardisoSolver: self._factored = True - cdef _solve(self, void* b, void* x, MKL_INT nrhs_in): + cdef _solve(self, void* b, void* x, int_t nrhs_in): #phase = 33 if(not self._factored): raise PardisoError("Cannot solve without a previous factorization.") @@ -571,9 +574,9 @@ cdef class MKLPardisoSolver: raise PardisoError("Solve step error, "+_err_messages[err]) @cython.boundscheck(False) - cdef MKL_INT _run_pardiso(self, MKL_INT phase, void* b=NULL, void* x=NULL, MKL_INT nrhs=0) nogil: - cdef MKL_INT error=0 - cdef MKL_INT64 error64=0, phase64=phase, nrhs64=nrhs + cdef int _run_pardiso(self, int_t phase, void* b=NULL, void* x=NULL, int_t nrhs=0) nogil: + cdef int_t error=0 + cdef long_t error64=0, phase64=phase, nrhs64=nrhs PyThread_acquire_lock(self.lock, 1) if self._call32: @@ -585,6 +588,6 @@ cdef class MKLPardisoSolver: &phase64, &self._par64.n, self.a, &self._par64.ia[0], &self._par64.ja[0], &self._par64.perm[0], &nrhs64, self._par64.iparm, &self._par64.msglvl, b, x, &error64) PyThread_release_lock(self.lock) - error = error or error64 + error = error or error64 return error From 5deb393acf22343f5d812eef48196fe6c6bd86c5 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 15:09:17 -0700 Subject: [PATCH 27/53] more type resetting --- pydiso/mkl_solver.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index b0bf486..d581c0c 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -49,7 +49,7 @@ cdef extern from 'mkl.h': void pardiso(_MKL_DSS_HANDLE_t, const int*, const int*, const int*, const int *, const int *, const void *, const int *, - const int *, int *, const int_t *, int *, + const int *, int *, const int *, int *, const int *, void *, void *, int *) nogil void pardiso_64(_MKL_DSS_HANDLE_t, const long_t *, const long_t *, const long_t *, @@ -192,9 +192,9 @@ cdef class MKLPardisoSolver: cdef _MKL_DSS_HANDLE_t handle[64] cdef _PardisoParams _par cdef _PardisoParams64 _par64 - cdef int _call32 - cdef int mat_type - cdef int _factored + cdef int_t _call32 + cdef int_t mat_type + cdef int_t _factored cdef size_t shape[2] cdef PyThread_type_lock lock cdef void * a @@ -443,7 +443,7 @@ cdef class MKLPardisoSolver: else: return np.array(self._par64.iparm) - def set_iparm(self, int i, int val): + def set_iparm(self, int_t i, int_t val): if i > 63 or i < 0: raise IndexError(f"index {i} is out of bounds for size 64 array") if i not in [ From 9c1ac26faac701878899bc9ea52ac9b27822b86e Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 15:18:34 -0700 Subject: [PATCH 28/53] unsetting some more --- pydiso/mkl_solver.pyx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index d581c0c..533f927 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -42,7 +42,7 @@ cdef extern from 'mkl.h': int mkl_get_max_threads() int mkl_domain_get_max_threads(int domain) - ctypedef int (*ProgressEntry)(int* thread, int* step, char* stage, int stage_len) except? -1; + ctypedef int (*ProgressEntry)(int_t* thread, int_t* step, char* stage, int_t stage_len) except? -1; ProgressEntry mkl_set_progress(ProgressEntry progress); ctypedef void * _MKL_DSS_HANDLE_t @@ -64,11 +64,11 @@ else: _np_mkl_int64 = np.int64 #call pardiso (pt, maxfct, mnum, mtype, phase, n, a, ia, ja, perm, nrhs, iparm, msglvl, b, x, error) -cdef int mkl_progress(int *thread, int* step, char* stage, int stage_len): +cdef int mkl_progress(int_t *thread, int_t* step, char* stage, int_t stage_len): print(thread[0], step[0], stage, stage_len) return 0 -cdef int mkl_no_progress(int *thread, int* step, char* stage, int stage_len) nogil: +cdef int mkl_no_progress(int_t *thread, int_t* step, char* stage, int_t stage_len) nogil: return 0 MATRIX_TYPES ={ @@ -308,9 +308,11 @@ cdef class MKLPardisoSolver: # sizeof(MKL_INT) == 4 and A.indices.itemsize == 8 self._call32 = not (sizeof(int_t) == 4 and integer_len == 8) if self._call32: + print("calling 32") self._par = _PardisoParams() self._initialize(self._par, A, matrix_type, verbose) elif integer_len == 8: + print("calling 64") self._par64 = _PardisoParams64() self._initialize(self._par64, A, matrix_type, verbose) else: @@ -588,6 +590,6 @@ cdef class MKLPardisoSolver: &phase64, &self._par64.n, self.a, &self._par64.ia[0], &self._par64.ja[0], &self._par64.perm[0], &nrhs64, self._par64.iparm, &self._par64.msglvl, b, x, &error64) PyThread_release_lock(self.lock) - error = error or error64 + error = error or error64 return error From 0303889d3a47c17c05b5fc25a2931729f2722ca6 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 15:24:15 -0700 Subject: [PATCH 29/53] make call type a little more accurate --- pydiso/mkl_solver.pyx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 533f927..a250829 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -306,17 +306,21 @@ cdef class MKLPardisoSolver: integer_len = A.indices.itemsize # we only need to call the 64 bit version if # sizeof(MKL_INT) == 4 and A.indices.itemsize == 8 - self._call32 = not (sizeof(int_t) == 4 and integer_len == 8) + if sizeof(int_t) == 4 and integer_len==8: + self._call32 = False + elif sizeof(int_t) == integer_len: + self._call32 = True + else: + raise PardisoError("Unrecognized integer length") + if self._call32: print("calling 32") self._par = _PardisoParams() self._initialize(self._par, A, matrix_type, verbose) - elif integer_len == 8: + else: print("calling 64") self._par64 = _PardisoParams64() self._initialize(self._par64, A, matrix_type, verbose) - else: - raise PardisoError("Unrecognized integer length") if verbose: #for reporting factorization progress via python's `print` From 41f85a42be1b6532a3c79210cb208567b528ea72 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 15:32:36 -0700 Subject: [PATCH 30/53] switch back to conda (with defaults) --- .github/workflows/python-package-conda.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 5309b45..08c2852 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -13,7 +13,7 @@ on: - cron: "0 13 * * 1" env: - CACHE_NUMBER: 0 # increase to reset cache manually + CACHE_NUMBER: 1 # increase to reset cache manually jobs: @@ -45,9 +45,9 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - mamba-version: '*' - channels: conda-forge, defaults - channel-priority: true + #mamba-version: '*' + #channels: conda-forge, defaults + #channel-priority: true activate-environment: dev - name: Get Date @@ -68,7 +68,7 @@ jobs: - name: Create environment run: | - mamba install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm + conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm if: steps.cache.outputs.cache-hit != 'true' - name: Install Our Package From a41b8d6eefc1aff01968d91ed758135e5670742c Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 15:38:24 -0700 Subject: [PATCH 31/53] reset some build scripts --- .github/workflows/python-package-conda.yml | 27 +++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 08c2852..0032ad6 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -40,25 +40,26 @@ jobs: prefix: C:\Miniconda3\envs\my-env steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - name: Setup Conda - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@v2 with: + auto-update-conda: true + activate-environment: dev python-version: ${{ matrix.python-version }} #mamba-version: '*' #channels: conda-forge, defaults #channel-priority: true - activate-environment: dev - - - name: Get Date - id: get-date - run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT - - uses: actions/cache@v2 - with: - path: ${{ matrix.prefix }} - key: ${{ matrix.label }}-${{ matrix.python-version }}-conda-${{ env.DATE }}-${{ env.CACHE_NUMBER }} - id: cache +# - name: Get Date +# id: get-date +# run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT +# +# - uses: actions/cache@v2 +# with: +# path: ${{ matrix.prefix }} +# key: ${{ matrix.label }}-${{ matrix.python-version }}-conda-${{ env.DATE }}-${{ env.CACHE_NUMBER }} +# id: cache - name: Conda information run: | @@ -69,7 +70,7 @@ jobs: - name: Create environment run: | conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm - if: steps.cache.outputs.cache-hit != 'true' + #if: steps.cache.outputs.cache-hit != 'true' - name: Install Our Package run: | From ab72d25ed9a42715298cdea8d1f913ba3d09acb3 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 16:47:47 -0700 Subject: [PATCH 32/53] switch back to mamba, but don't use mkl 2024 --- .github/workflows/python-package-conda.yml | 39 +++++++++++----------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 0032ad6..bda10cf 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -13,7 +13,7 @@ on: - cron: "0 13 * * 1" env: - CACHE_NUMBER: 1 # increase to reset cache manually + CACHE_NUMBER: 0 # increase to reset cache manually jobs: @@ -40,26 +40,25 @@ jobs: prefix: C:\Miniconda3\envs\my-env steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: - auto-update-conda: true - activate-environment: dev python-version: ${{ matrix.python-version }} - #mamba-version: '*' - #channels: conda-forge, defaults - #channel-priority: true - -# - name: Get Date -# id: get-date -# run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT -# -# - uses: actions/cache@v2 -# with: -# path: ${{ matrix.prefix }} -# key: ${{ matrix.label }}-${{ matrix.python-version }}-conda-${{ env.DATE }}-${{ env.CACHE_NUMBER }} -# id: cache + mamba-version: '*' + channels: conda-forge, defaults + channel-priority: true + activate-environment: dev + + - name: Get Date + id: get-date + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + + - uses: actions/cache@v2 + with: + path: ${{ matrix.prefix }} + key: ${{ matrix.label }}-${{ matrix.python-version }}-conda-${{ env.DATE }}-${{ env.CACHE_NUMBER }} + id: cache - name: Conda information run: | @@ -69,8 +68,8 @@ jobs: - name: Create environment run: | - conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm - #if: steps.cache.outputs.cache-hit != 'true' + mamba install --quiet --yes pip numpy scipy cython mkl=2023 pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm + if: steps.cache.outputs.cache-hit != 'true' - name: Install Our Package run: | From 8765c599198d3e5f226f5f95faf14778334c1b4a Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 16:48:10 -0700 Subject: [PATCH 33/53] ensure cleared cache --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index bda10cf..845470c 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -13,7 +13,7 @@ on: - cron: "0 13 * * 1" env: - CACHE_NUMBER: 0 # increase to reset cache manually + CACHE_NUMBER: 3 # increase to reset cache manually jobs: From 9e72e6ea548b10bbe4e86fa605192b1d04f28688 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 16:50:56 -0700 Subject: [PATCH 34/53] retry this --- pydiso/mkl_solver.pyx | 96 ++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 61 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index a250829..b0dbfac 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -16,9 +16,10 @@ import numpy as np import scipy.sparse as sp import os -ctypedef long long MKL_INT64 -ctypedef unsigned long long MKL_UINT64 -ctypedef int MKL_INT +cdef extern from 'mkl.h': + ctypedef long long MKL_INT64 + ctypedef unsigned long long MKL_UINT64 + ctypedef int MKL_INT ctypedef MKL_INT int_t ctypedef MKL_INT64 long_t @@ -42,33 +43,28 @@ cdef extern from 'mkl.h': int mkl_get_max_threads() int mkl_domain_get_max_threads(int domain) - ctypedef int (*ProgressEntry)(int_t* thread, int_t* step, char* stage, int_t stage_len) except? -1; + ctypedef int (*ProgressEntry)(int* thread, int* step, char* stage, int stage_len) except? -1; ProgressEntry mkl_set_progress(ProgressEntry progress); ctypedef void * _MKL_DSS_HANDLE_t - void pardiso(_MKL_DSS_HANDLE_t, const int*, const int*, const int*, - const int *, const int *, const void *, const int *, - const int *, int *, const int *, int *, - const int *, void *, void *, int *) nogil + void pardiso(_MKL_DSS_HANDLE_t, const int_t*, const int_t*, const int_t*, + const int_t *, const int_t *, const void *, const int_t *, + const int_t *, int_t *, const int_t *, int_t *, + const int_t *, void *, void *, int_t *) nogil void pardiso_64(_MKL_DSS_HANDLE_t, const long_t *, const long_t *, const long_t *, const long_t *, const long_t *, const void *, const long_t *, const long_t *, long_t *, const long_t *, long_t *, const long_t *, void *, void *, long_t *) nogil -if sizeof(int_t) == 4: - _np_mkl_int = np.int32 -else: - _np_mkl_int = np.int64 -_np_mkl_int64 = np.int64 #call pardiso (pt, maxfct, mnum, mtype, phase, n, a, ia, ja, perm, nrhs, iparm, msglvl, b, x, error) -cdef int mkl_progress(int_t *thread, int_t* step, char* stage, int_t stage_len): +cdef int mkl_progress(int *thread, int* step, char* stage, int stage_len): print(thread[0], step[0], stage, stage_len) return 0 -cdef int mkl_no_progress(int_t *thread, int_t* step, char* stage, int_t stage_len) nogil: +cdef int mkl_no_progress(int *thread, int* step, char* stage, int stage_len) nogil: return 0 MATRIX_TYPES ={ @@ -192,10 +188,11 @@ cdef class MKLPardisoSolver: cdef _MKL_DSS_HANDLE_t handle[64] cdef _PardisoParams _par cdef _PardisoParams64 _par64 - cdef int_t _call32 + cdef int_t _is_32 cdef int_t mat_type cdef int_t _factored cdef size_t shape[2] + cdef int_t _initialized cdef PyThread_type_lock lock cdef void * a @@ -203,13 +200,8 @@ cdef class MKLPardisoSolver: cdef object _Adata #a reference to make sure the pointer "a" doesn't get destroyed def __cinit__(self, *args, **kwargs): - # allocate the lock self.lock = PyThread_allocate_lock() - # handle should be intialized just fine, but just in case.. - for i in range(64): - self.handle[64] = NULL - def __init__(self, A, matrix_type=None, factor=True, verbose=False): '''ParidsoSolver(A, matrix_type=None, factor=True, verbose=False) An interface to the intel MKL pardiso sparse matrix solver. @@ -267,6 +259,7 @@ cdef class MKLPardisoSolver: >>> np.allclose(x, x_solved) True ''' + self._initialized = False n_row, n_col = A.shape if n_row != n_col: raise ValueError("Matrix is not square") @@ -304,25 +297,17 @@ cdef class MKLPardisoSolver: #set integer length integer_len = A.indices.itemsize - # we only need to call the 64 bit version if - # sizeof(MKL_INT) == 4 and A.indices.itemsize == 8 - if sizeof(int_t) == 4 and integer_len==8: - self._call32 = False - elif sizeof(int_t) == integer_len: - self._call32 = True - else: - raise PardisoError("Unrecognized integer length") + #self._is_32 = integer_len == sizeof(int_t) + self._is_32 = sizeof(int_t) == 8 or integer_len == sizeof(int_t) - if self._call32: - print("calling 32") + if self._is_32: self._par = _PardisoParams() self._initialize(self._par, A, matrix_type, verbose) else: - print("calling 64") self._par64 = _PardisoParams64() self._initialize(self._par64, A, matrix_type, verbose) - if verbose: + if(verbose): #for reporting factorization progress via python's `print` mkl_set_progress(mkl_progress) else: @@ -330,6 +315,7 @@ cdef class MKLPardisoSolver: self._set_A(A.data) self._analyze() + self._initialized = True self._factored = False if factor: self._factor() @@ -362,14 +348,6 @@ cdef class MKLPardisoSolver: def __call__(self, b): return self.solve(b) - cpdef int _initialized(self): - # this says if the handle was ever initialized - cdef int i - for i in range(64): - if self.handle[i]: - return 1 - return 0 - def solve(self, b, x=None, transpose=False): """solve(self, b, x=None, transpose=False) Solves the equation AX=B using the factored A matrix @@ -435,7 +413,7 @@ cdef class MKLPardisoSolver: def perm(self): """ Fill-reducing permutation vector used inside pardiso. """ - if self._call32: + if self._is_32: return np.array(self._par.perm) else: return np.array(self._par64.perm) @@ -444,7 +422,7 @@ cdef class MKLPardisoSolver: def iparm(self): """ Parameter options for the pardiso solver. """ - if self._call32: + if self._is_32: return np.array(self._par.iparm) else: return np.array(self._par64.iparm) @@ -457,7 +435,7 @@ cdef class MKLPardisoSolver: 24, 26, 30, 33, 34, 35, 36, 38, 42, 55, 59 ]: raise PardisoError(f"cannot set parameter {i} of the iparm array") - if self._call32: + if self._is_32: self._par.iparm[i] = val else: self._par64.iparm[i] = val @@ -467,12 +445,13 @@ cdef class MKLPardisoSolver: return self.iparm[17] cdef _initialize(self, _par_params par, A, matrix_type, verbose): + if _par_params is _PardisoParams: - np_int_dtype = _np_mkl_int + int_dtype = f'i{sizeof(int_t)}' else: - np_int_dtype = _np_mkl_int64 + int_dtype = 'i8' par.n = A.shape[0] - par.perm = np.empty(par.n, dtype=np_int_dtype) + par.perm = np.empty(par.n, dtype=int_dtype) par.maxfct = 1 par.mnum = 1 @@ -495,7 +474,7 @@ cdef class MKLPardisoSolver: par.iparm[18] = 0 # do not report flop count par.iparm[20] = 1 if matrix_type in [-2, -4, 6] else 0 par.iparm[23] = 0 # classic (not parallel) factorization - par.iparm[24] = 0 # default behavior of parallel solving + par.iparm[24] = 0 # default behavoir of parallel solving par.iparm[26] = 0 # Do not check the input matrix #set precision if self._data_type==np.float64 or self._data_type==np.complex128: @@ -514,22 +493,21 @@ cdef class MKLPardisoSolver: par.iparm[55] = 0 # Internal function used to work with pivot and calculation of diagonal arrays turned off. par.iparm[59] = 0 # operate in-core mode - par.ia = np.require(A.indptr, dtype=np_int_dtype) - par.ja = np.require(A.indices, dtype=np_int_dtype) + par.ia = np.require(A.indptr, dtype=int_dtype) + par.ja = np.require(A.indices, dtype=int_dtype) cdef _set_A(self, data): self._Adata = data self.a = np.PyArray_DATA(data) - def __del__(self): + def __dealloc__(self): # Need to call pardiso with phase=-1 to release memory cdef int_t phase=-1, nrhs=0, error=0 cdef long_t phase64=-1, nrhs64=0, error64=0 - # Only need to deallocate if the handle itself was ever allocated. - if self._initialized(): + if self._initialized: PyThread_acquire_lock(self.lock, 1) - if self._call32: + if self._is_32: pardiso( self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, &phase, &self._par.n, self.a, NULL, NULL, NULL, &nrhs, self._par.iparm, @@ -544,14 +522,11 @@ cdef class MKLPardisoSolver: PyThread_release_lock(self.lock) err = error or error64 if err!=0: - raise PardisoError("Memmory release error "+_err_messages[err]) - + raise PardisoError("Memory release error "+_err_messages[err]) - def __dealloc__(self): if self.lock: #dealloc lock PyThread_free_lock(self.lock) - self.lock = NULL cdef _analyze(self): #phase = 11 @@ -585,7 +560,7 @@ cdef class MKLPardisoSolver: cdef long_t error64=0, phase64=phase, nrhs64=nrhs PyThread_acquire_lock(self.lock, 1) - if self._call32: + if self._is_32: pardiso(self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, &phase, &self._par.n, self.a, &self._par.ia[0], &self._par.ja[0], &self._par.perm[0], &nrhs, self._par.iparm, &self._par.msglvl, b, x, &error) @@ -595,5 +570,4 @@ cdef class MKLPardisoSolver: &self._par64.perm[0], &nrhs64, self._par64.iparm, &self._par64.msglvl, b, x, &error64) PyThread_release_lock(self.lock) error = error or error64 - return error - + return error \ No newline at end of file From 6906999ec0c0adc20d0d5fb477685cdf64235053 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 16:56:30 -0700 Subject: [PATCH 35/53] dynamic initialized (again) --- pydiso/mkl_solver.pyx | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index b0dbfac..618003b 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -192,12 +192,11 @@ cdef class MKLPardisoSolver: cdef int_t mat_type cdef int_t _factored cdef size_t shape[2] - cdef int_t _initialized cdef PyThread_type_lock lock cdef void * a cdef object _data_type - cdef object _Adata #a reference to make sure the pointer "a" doesn't get destroyed + cdef object _Adata # a reference to make sure the pointer "a" doesn't get destroyed def __cinit__(self, *args, **kwargs): self.lock = PyThread_allocate_lock() @@ -259,7 +258,6 @@ cdef class MKLPardisoSolver: >>> np.allclose(x, x_solved) True ''' - self._initialized = False n_row, n_col = A.shape if n_row != n_col: raise ValueError("Matrix is not square") @@ -307,7 +305,7 @@ cdef class MKLPardisoSolver: self._par64 = _PardisoParams64() self._initialize(self._par64, A, matrix_type, verbose) - if(verbose): + if verbose: #for reporting factorization progress via python's `print` mkl_set_progress(mkl_progress) else: @@ -315,7 +313,6 @@ cdef class MKLPardisoSolver: self._set_A(A.data) self._analyze() - self._initialized = True self._factored = False if factor: self._factor() @@ -345,6 +342,13 @@ cdef class MKLPardisoSolver: self._set_A(A.data) self._factor() + cdef _initialized(self): + cdef int i + for i in range(64): + if self.handle[i]: + return 1 + return 0 + def __call__(self, b): return self.solve(b) @@ -505,18 +509,18 @@ cdef class MKLPardisoSolver: cdef int_t phase=-1, nrhs=0, error=0 cdef long_t phase64=-1, nrhs64=0, error64=0 - if self._initialized: + if self._initialized(): PyThread_acquire_lock(self.lock, 1) if self._is_32: pardiso( self.handle, &self._par.maxfct, &self._par.mnum, &self._par.mtype, - &phase, &self._par.n, self.a, NULL, NULL, NULL, &nrhs, self._par.iparm, + &phase, &self._par.n, NULL, NULL, NULL, NULL, &nrhs, self._par.iparm, &self._par.msglvl, NULL, NULL, &error ) else: pardiso_64( self.handle, &self._par64.maxfct, &self._par64.mnum, &self._par64.mtype, - &phase64, &self._par64.n, self.a, NULL, NULL, NULL, &nrhs64, + &phase64, &self._par64.n, NULL, NULL, NULL, NULL, &nrhs64, self._par64.iparm, &self._par64.msglvl, NULL, NULL, &error64 ) PyThread_release_lock(self.lock) From 6f03c16519a0c4fcdb2d92420a743247a9902a00 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 16:59:40 -0700 Subject: [PATCH 36/53] be explicit about initializing handle and iparm --- pydiso/mkl_solver.pyx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 618003b..5163034 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -201,6 +201,9 @@ cdef class MKLPardisoSolver: def __cinit__(self, *args, **kwargs): self.lock = PyThread_allocate_lock() + for i in range(64): + self.handle[i] = NULL + def __init__(self, A, matrix_type=None, factor=True, verbose=False): '''ParidsoSolver(A, matrix_type=None, factor=True, verbose=False) An interface to the intel MKL pardiso sparse matrix solver. @@ -463,6 +466,9 @@ cdef class MKLPardisoSolver: par.mtype = matrix_type par.msglvl = verbose + for i in range(64): + par.iparm[i] = 0 # ensure these all start at 0 + # set default parameters par.iparm[0] = 1 # tell pardiso to not reset these values on the first call par.iparm[1] = 2 # The nested dissection algorithm from the METIS @@ -527,6 +533,8 @@ cdef class MKLPardisoSolver: err = error or error64 if err!=0: raise PardisoError("Memory release error "+_err_messages[err]) + for i in range(64): + self.handle[i] = NULL if self.lock: #dealloc lock From 25ac2f589935bb88fd84e006de9aeb1dbbe0e0fa Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Fri, 1 Mar 2024 17:01:53 -0700 Subject: [PATCH 37/53] don't cache --- .github/workflows/python-package-conda.yml | 25 ---------------------- 1 file changed, 25 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 845470c..7665a88 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -12,10 +12,6 @@ on: schedule: - cron: "0 13 * * 1" -env: - CACHE_NUMBER: 3 # increase to reset cache manually - - jobs: build-and-test: name: Testing (${{ matrix.python-version }}, ${{ matrix.os }}) @@ -28,16 +24,6 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - include: - - os: ubuntu-latest - label: linux-64 - prefix: /usr/share/miniconda3/envs/my-env - - os: macos-latest - label: osx-64 - prefix: /usr/share/miniconda3/envs/my-env - - os: windows-latest - label: win-64 - prefix: C:\Miniconda3\envs\my-env steps: - uses: actions/checkout@v4 @@ -50,16 +36,6 @@ jobs: channel-priority: true activate-environment: dev - - name: Get Date - id: get-date - run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT - - - uses: actions/cache@v2 - with: - path: ${{ matrix.prefix }} - key: ${{ matrix.label }}-${{ matrix.python-version }}-conda-${{ env.DATE }}-${{ env.CACHE_NUMBER }} - id: cache - - name: Conda information run: | conda info @@ -69,7 +45,6 @@ jobs: - name: Create environment run: | mamba install --quiet --yes pip numpy scipy cython mkl=2023 pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm - if: steps.cache.outputs.cache-hit != 'true' - name: Install Our Package run: | From 207dc8ea4c230c5efe54e801fe919d54ba0b7f7e Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 11 Mar 2024 14:23:07 -0600 Subject: [PATCH 38/53] generate c sources in tree for cython coverage --- .coveragerc | 2 -- .github/workflows/python-package-conda.yml | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index aca8f6b..bb3cf2c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,3 @@ [run] source = pydiso plugins = Cython.Coverage -omit = - */setup.py diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 7665a88..8d8c16b 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -46,6 +46,11 @@ jobs: run: | mamba install --quiet --yes pip numpy scipy cython mkl=2023 pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm + - name: Generate cython c # sources for coverage + run: | + cd pydiso + cython mkl_solver.pyx + - name: Install Our Package run: | python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v From 00435b1d8f12653bdc129a3a93866be6d7eea8be Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 11 Mar 2024 14:23:55 -0600 Subject: [PATCH 39/53] enable coverage build option --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 8d8c16b..fd5afa3 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -53,7 +53,7 @@ jobs: - name: Install Our Package run: | - python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v + python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v --config-settings=setup-args="-Dcy_line_trace=true" conda list - name: Run Tests From 29bd20a0c86a121fdd57ca302ae1db00b0313c66 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 11 Mar 2024 14:45:44 -0600 Subject: [PATCH 40/53] have meson conditionally generate c file in source directory when doing coverage. --- .github/workflows/python-package-conda.yml | 5 ----- pydiso/meson.build | 2 ++ 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index fd5afa3..68f5930 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -46,11 +46,6 @@ jobs: run: | mamba install --quiet --yes pip numpy scipy cython mkl=2023 pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm - - name: Generate cython c # sources for coverage - run: | - cd pydiso - cython mkl_solver.pyx - - name: Install Our Package run: | python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v --config-settings=setup-args="-Dcy_line_trace=true" diff --git a/pydiso/meson.build b/pydiso/meson.build index 0b4d738..a4b06d6 100644 --- a/pydiso/meson.build +++ b/pydiso/meson.build @@ -95,6 +95,8 @@ cython_c_args = [numpy_nodepr_api, use_math_defines] cy_line_trace = get_option('cy_line_trace') if cy_line_trace cython_c_args += ['-DCYTHON_TRACE_NOGIL=1'] + cython_file = join_paths(meson.current_source_dir(), 'mkl_solver.pyx') + run_command('cython', cython_file) endif cython_cpp_args = cython_c_args From 24498d6bd99c27537949bb147fbb69662f003ed0 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 11 Mar 2024 16:38:17 -0600 Subject: [PATCH 41/53] add coverage configuration to pyproject.toml --- .coveragerc | 3 -- .github/workflows/python-package-conda.yml | 12 ++++--- meson.options | 2 +- pydiso/meson.build | 9 +++-- pydiso/mkl_solver.pyx | 1 - pyproject.toml | 42 +++++++++++++++------- 6 files changed, 44 insertions(+), 25 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index bb3cf2c..0000000 --- a/.coveragerc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -source = pydiso -plugins = Cython.Coverage diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 68f5930..27ddb7e 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -24,6 +24,10 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + include: + - os: ubuntu-latest + version: "3.12" + coverage: true steps: - uses: actions/checkout@v4 @@ -44,19 +48,19 @@ jobs: - name: Create environment run: | - mamba install --quiet --yes pip numpy scipy cython mkl=2023 pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm + mamba install --quiet --yes pip numpy scipy cython mkl=2023 pytest mkl-devel pkg-config meson-python meson ninja setuptools_scm ${{ matrix.coverage && 'coverage' || ''}} - name: Install Our Package run: | - python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v --config-settings=setup-args="-Dcy_line_trace=true" + python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v ${{ matrix.coverage && '--config-settings=setup-args="-Dcy_coverage=true"' || ''}} conda list - name: Run Tests run: | - pytest --cov-config=.coveragerc --cov-report=xml --cov=pydiso -s -v + ${{ matrix.coverage && 'coverage run -m' || '' }} pytest --cov-config=.coveragerc --cov-report=xml --cov=pydiso -s -v - name: Upload coverage - if: ${{ matrix.os == 'ubuntu-latest' }} and {{ matrix.python-version == '3.8' }} + if: ${{ matrix.coverage }} uses: codecov/codecov-action@v2 with: verbose: true # optional (default = false) diff --git a/meson.options b/meson.options index 6f73493..f3968b5 100644 --- a/meson.options +++ b/meson.options @@ -1,4 +1,4 @@ -option('cy_line_trace', type : 'boolean', value : false) +option('cy_coverage', type : 'boolean', value : false) option('use-sdl', type: 'boolean', value: true, description: 'Use the single dynamic library.') diff --git a/pydiso/meson.build b/pydiso/meson.build index a4b06d6..2bc60aa 100644 --- a/pydiso/meson.build +++ b/pydiso/meson.build @@ -92,11 +92,13 @@ c_undefined_ok = ['-Wno-maybe-uninitialized'] cython_c_args = [numpy_nodepr_api, use_math_defines] -cy_line_trace = get_option('cy_line_trace') -if cy_line_trace +cython_args = [] + +if get_option('cy_coverage') cython_c_args += ['-DCYTHON_TRACE_NOGIL=1'] + cython_args += ['--directive', 'linetrace=true'] cython_file = join_paths(meson.current_source_dir(), 'mkl_solver.pyx') - run_command('cython', cython_file) + run_command('cython', '--directive linetrace=true', cython_file) endif cython_cpp_args = cython_c_args @@ -107,6 +109,7 @@ py.extension_module( 'mkl_solver', 'mkl_solver.pyx', include_directories: incdir_numpy, + cython_args: cython_args, c_args: cython_c_args, install: true, subdir: module_path, diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index 5163034..bfe2063 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -1,5 +1,4 @@ #cython: language_level=3 -#cython: linetrace=True cimport numpy as np import cython from cython cimport numeric diff --git a/pyproject.toml b/pyproject.toml index e93252c..0178128 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,16 +93,32 @@ Repository = 'https://github.com/simpeg/pydiso.git' [tool.setuptools_scm] -[tool.meson-python.args] -# for windows this will prioritize the msvc compilers -setup = ['--vsenv'] - -[tool.cibuildwheel] -# skip building wheels for python 3.6, 3.7, all pypy versions, and specialty linux -# processors (still does arm builds though). -# skip windows 32bit -skip = "cp36-* cp37-* pp* *_ppc64le *_i686 *_s390x *-win32 cp38-musllinux_x86_64" -build-verbosity = "3" - -# test importing discretize to make sure externals are loadable. -test-command = 'python -c "import pydiso; print(pydiso.__version__)"' \ No newline at end of file +[tool.coverage.run] +branch = true +plugins = ["Cython.Coverage"] +source = ["pydiso", "tests"] + +[tool.coverage.report] +ignore_errors = false +show_missing = true +# Regexes for lines to exclude from consideration +exclude_also = [ + # Don't complain about missing debug-only code: + "def __repr__", + "if self\\.debug", + + # Don't complain if tests don't hit defensive assertion code: + "raise AssertionError", + "raise NotImplementedError", + "AbstractMethodError", + + # Don't complain if non-runnable code isn't run: + "if 0:", + "if __name__ == .__main__.:", + + # Don't complain about abstract methods, they aren't run: + "@(abc\\.)?abstractmethod", +] + +[tool.coverage.html] +directory = "coverage_html_report" \ No newline at end of file From a8e5418eb2884aa13259097725bc8bea2b69d886 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 11 Mar 2024 16:46:43 -0600 Subject: [PATCH 42/53] remove all pytest-cov related options to pytest --- .github/workflows/python-package-conda.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 27ddb7e..1c04d76 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -27,7 +27,7 @@ jobs: include: - os: ubuntu-latest version: "3.12" - coverage: true + coverage: ${{ true }} steps: - uses: actions/checkout@v4 @@ -52,12 +52,12 @@ jobs: - name: Install Our Package run: | - python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v ${{ matrix.coverage && '--config-settings=setup-args="-Dcy_coverage=true"' || ''}} + python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v ${{ matrix.coverage && '--config-settings=setup-args="-Dcy_coverage=true"' || ' '}} conda list - name: Run Tests run: | - ${{ matrix.coverage && 'coverage run -m' || '' }} pytest --cov-config=.coveragerc --cov-report=xml --cov=pydiso -s -v + ${{ matrix.coverage && 'coverage run -m' || ' ' }}pytest -s -v - name: Upload coverage if: ${{ matrix.coverage }} From d1a0be5451a46f1e849c9c6b2b4191263fcc6970 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 11 Mar 2024 16:57:05 -0600 Subject: [PATCH 43/53] fix coverage determination, split long lines --- .github/workflows/python-package-conda.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 1c04d76..4a7417b 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -26,7 +26,7 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] include: - os: ubuntu-latest - version: "3.12" + python-version: "3.12" coverage: ${{ true }} steps: @@ -48,16 +48,20 @@ jobs: - name: Create environment run: | - mamba install --quiet --yes pip numpy scipy cython mkl=2023 pytest mkl-devel pkg-config meson-python meson ninja setuptools_scm ${{ matrix.coverage && 'coverage' || ''}} + mamba install --quiet --yes pip numpy scipy cython mkl=2023 pytest \ + mkl-devel pkg-config meson-python meson ninja setuptools_scm \ + ${{ matrix.coverage && 'coverage' || ''}} - name: Install Our Package run: | - python -m pip install --no-build-isolation --verbose --editable . --config-setting=compile-args=-v ${{ matrix.coverage && '--config-settings=setup-args="-Dcy_coverage=true"' || ' '}} + python -m pip install --no-build-isolation --verbose --editable . \ + --config-setting=compile-args=-v \ + ${{ matrix.coverage && '--config-settings=setup-args="-Dcy_coverage=true"' || ''}} conda list - name: Run Tests run: | - ${{ matrix.coverage && 'coverage run -m' || ' ' }}pytest -s -v + ${{ matrix.coverage && 'coverage run -m' || '' }} pytest -s -v - name: Upload coverage if: ${{ matrix.coverage }} From 596ed7a7a1c3354e87e3cdc8d8afb9917daf8677 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 11 Mar 2024 17:17:14 -0600 Subject: [PATCH 44/53] generate coverage.xml --- .github/workflows/python-package-conda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 4a7417b..a4111db 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -62,10 +62,11 @@ jobs: - name: Run Tests run: | ${{ matrix.coverage && 'coverage run -m' || '' }} pytest -s -v + ${{ matrix.coverage && 'coverage xml' || '' }} - name: Upload coverage if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v4 with: verbose: true # optional (default = false) From 2aeceaf3d4ff0042bf81a0105b3be88e833db7e5 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Mon, 11 Mar 2024 23:10:18 -0600 Subject: [PATCH 45/53] fix command --- pydiso/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydiso/meson.build b/pydiso/meson.build index 2bc60aa..3c6576c 100644 --- a/pydiso/meson.build +++ b/pydiso/meson.build @@ -98,7 +98,7 @@ if get_option('cy_coverage') cython_c_args += ['-DCYTHON_TRACE_NOGIL=1'] cython_args += ['--directive', 'linetrace=true'] cython_file = join_paths(meson.current_source_dir(), 'mkl_solver.pyx') - run_command('cython', '--directive linetrace=true', cython_file) + run_command('cython', '--directive', 'linetrace=true', cython_file) endif cython_cpp_args = cython_c_args From cab5d60533506daaf88602520024545e5d45626c Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 13 Mar 2024 12:43:16 -0600 Subject: [PATCH 46/53] add comments to meson build file --- pydiso/meson.build | 22 ++++++++++++---------- pydiso/mkl_solver.pyx | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pydiso/meson.build b/pydiso/meson.build index 3c6576c..fd7f01c 100644 --- a/pydiso/meson.build +++ b/pydiso/meson.build @@ -90,26 +90,28 @@ endif numpy_nodepr_api = '-DNPY_NO_DEPRECATED_API=NPY_1_9_API_VERSION' c_undefined_ok = ['-Wno-maybe-uninitialized'] -cython_c_args = [numpy_nodepr_api, use_math_defines] +cython_c_args = [numpy_nodepr_api, use_math_defines]\ -cython_args = [] +cython_file = 'mkl_solver.pyx' +cython_file_full_path = meson.current_source_dir() / cython_file if get_option('cy_coverage') - cython_c_args += ['-DCYTHON_TRACE_NOGIL=1'] - cython_args += ['--directive', 'linetrace=true'] - cython_file = join_paths(meson.current_source_dir(), 'mkl_solver.pyx') - run_command('cython', '--directive', 'linetrace=true', cython_file) + # tell cython to enable linetracing + add_project_arguments(['--directive', 'linetrace=true'], language : 'cython') + # tell the c_compiler to definie the CYTHON_TRACE_NOGIL + add_project_arguments(['-DCYTHON_TRACE_NOGIL=1'], language : 'c') + + # compile the .c file from the .pyx file in it's directory. + # These should include the default options passed to the cython compiler + run_command(cython, '-M', '--fast-fail', '-3', '--directive', 'linetrace=true', cython_file_full_path) endif -cython_cpp_args = cython_c_args - module_path = 'pydiso' py.extension_module( 'mkl_solver', - 'mkl_solver.pyx', + cython_file, include_directories: incdir_numpy, - cython_args: cython_args, c_args: cython_c_args, install: true, subdir: module_path, diff --git a/pydiso/mkl_solver.pyx b/pydiso/mkl_solver.pyx index bfe2063..5d0a64e 100644 --- a/pydiso/mkl_solver.pyx +++ b/pydiso/mkl_solver.pyx @@ -112,7 +112,7 @@ def _ensure_csr(A, sym=False): if sym and sp.isspmatrix_csc(A): A = A.T else: - warnings.warn("Converting %s matrix to CSR format, will slow down." + warnings.warn("Converting %s matrix to CSR format." %A.__class__.__name__, PardisoTypeConversionWarning) A = A.tocsr() return A From f492847bbbfd7271ef769a25f5b72c7bbaab7c08 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 13 Mar 2024 13:00:38 -0600 Subject: [PATCH 47/53] update distribution script --- .git_archival.txt | 4 ++++ .gitattributes | 11 +++++++++++ .github/workflows/python-package-conda.yml | 20 +++++++++++--------- MANIFEST.in | 3 --- 4 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 .git_archival.txt create mode 100644 .gitattributes delete mode 100644 MANIFEST.in diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 0000000..b1a286b --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,4 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ +ref-names: $Format:%D$ \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e834cfc --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +.git_archival.txt export-subst +# Excluding files from an sdist generated by meson-python + +.azure-pipelines/* export-ignore +.ci/* export-ignore +tests/* export-ignore + +.flake8 export-ignore +.git* export-ignore +*.yml export-ignore +*.yaml export-ignore \ No newline at end of file diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index a4111db..4d64c1a 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -71,7 +71,7 @@ jobs: verbose: true # optional (default = false) distribute: - name: Distributing from 3.8 + name: Distributing from 3.11 needs: build-and-test if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') runs-on: ubuntu-latest @@ -80,25 +80,27 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: - auto-update-conda: true + python-version: 3.11 + mamba-version: '*' + channels: conda-forge, defaults + channel-priority: true activate-environment: dev - python-version: 3.8 - name: Install Env run: | conda info conda list conda config --show - conda install --quiet --yes pip numpy scipy cython mkl pytest pytest-cov mkl-devel pkg-config meson-python meson ninja setuptools_scm - - name: Install Our Package + - name: Create environment run: | - echo $CONDA_PREFIX - pip install -v -e . + mamba install --quiet --yes pip numpy scipy cython mkl=2023 \ + mkl-devel pkg-config meson-python meson ninja setuptools_scm \ + python-build - name: Generate Source Distribution run: | diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 9bdc9a3..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include *.rst LICENSE -global-include *.pyx -global-include *.pxd From ed93d8efe8dc7dbc163aa5382903f4a9ebfb83cb Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 13 Mar 2024 13:03:50 -0600 Subject: [PATCH 48/53] update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4100ca3..72fc51b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ build +dist +coverage_html_report *.so pydiso.egg-info __pycache__ mkl_solver.c +mkl_solver.c.dep .coverage coverage.xml From ef07d2074f9993ab50dbf408053d3f6adc9d8c9c Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 13 Mar 2024 13:08:43 -0600 Subject: [PATCH 49/53] add a test for mkl=2024 --- .github/workflows/python-package-conda.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 4d64c1a..3bc96ee 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -24,10 +24,14 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + mkl-version: ['2023'] # currently 2024 fails building for some reason... include: - os: ubuntu-latest python-version: "3.12" coverage: ${{ true }} + - os: ubuntu-latest + python-version: "3.12" + mkl-version: '2024' steps: - uses: actions/checkout@v4 @@ -48,7 +52,7 @@ jobs: - name: Create environment run: | - mamba install --quiet --yes pip numpy scipy cython mkl=2023 pytest \ + mamba install --quiet --yes pip numpy scipy cython mkl=${{ matrix.mkl_version }} pytest \ mkl-devel pkg-config meson-python meson ninja setuptools_scm \ ${{ matrix.coverage && 'coverage' || ''}} From 0fbd9b4581a94c8fd452a5f1beccf4930f07f0a3 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 13 Mar 2024 13:11:36 -0600 Subject: [PATCH 50/53] update test name --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 3bc96ee..ed3b754 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -14,7 +14,7 @@ on: jobs: build-and-test: - name: Testing (${{ matrix.python-version }}, ${{ matrix.os }}) + name: Testing (Python ${{ matrix.python-version }}, on ${{ matrix.os }}, with MKL ${{ matrix.mkl-version }}) runs-on: ${{ matrix.os }} defaults: run: From 3ba4c88320bee4d59737ce9f0decd0912310eece Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 13 Mar 2024 13:13:19 -0600 Subject: [PATCH 51/53] fix matrix variable name --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index ed3b754..f5dcf7a 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -52,7 +52,7 @@ jobs: - name: Create environment run: | - mamba install --quiet --yes pip numpy scipy cython mkl=${{ matrix.mkl_version }} pytest \ + mamba install --quiet --yes pip numpy scipy cython mkl=${{ matrix.mkl-version }} pytest \ mkl-devel pkg-config meson-python meson ninja setuptools_scm \ ${{ matrix.coverage && 'coverage' || ''}} From de6d10b56a55abe3621b39bf60bd56b3cc53b20d Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 13 Mar 2024 13:16:40 -0600 Subject: [PATCH 52/53] add test for windows and mac on mkl 2024 --- .github/workflows/python-package-conda.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index f5dcf7a..f0f28dc 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -32,6 +32,12 @@ jobs: - os: ubuntu-latest python-version: "3.12" mkl-version: '2024' + - os: windows-latest + python-version: "3.12" + mkl-version: '2024' + - os: macos-latest + python-version: "3.12" + mkl-version: '2024' steps: - uses: actions/checkout@v4 From 953be5ec3427a44583299c0d5040720674f73109 Mon Sep 17 00:00:00 2001 From: Joseph Capriotti Date: Wed, 13 Mar 2024 13:23:42 -0600 Subject: [PATCH 53/53] mkl-2024 is not on macos --- .github/workflows/python-package-conda.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index f0f28dc..925d788 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -35,9 +35,6 @@ jobs: - os: windows-latest python-version: "3.12" mkl-version: '2024' - - os: macos-latest - python-version: "3.12" - mkl-version: '2024' steps: - uses: actions/checkout@v4