From 286ff4a3f94eac2ebe3c73e74623946b2f6f22ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 18:20:10 +0000 Subject: [PATCH 01/12] chore(deps): bump selenium from 4.16.0 to 4.17.2 Bumps [selenium](https://github.com/SeleniumHQ/Selenium) from 4.16.0 to 4.17.2. - [Release notes](https://github.com/SeleniumHQ/Selenium/releases) - [Commits](https://github.com/SeleniumHQ/Selenium/commits) --- updated-dependencies: - dependency-name: selenium dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7ae9c728..22ff1806 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ mat73==0.62 torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 -selenium==4.16.0 +selenium==4.17.2 skl2onnx==1.16.0 onnx==1.15.0 onnxruntime==1.16.3 From 573ab073bc6dfbc462692385203201ee0b0bede3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 18:20:19 +0000 Subject: [PATCH 02/12] chore(deps): bump optuna from 3.4.0 to 3.5.0 Bumps [optuna](https://github.com/optuna/optuna) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/optuna/optuna/releases) - [Commits](https://github.com/optuna/optuna/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: optuna dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7ae9c728..74b90e1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ matplotlib==3.8.2 natsort==8.4.0 numpy==1.26.3 opencv_contrib_python==4.8.1.78 -optuna==3.4.0 +optuna==3.5.0 Pillow==10.2.0 pytest==7.4.4 scikit_learn==1.4.0 From b3fa65405aba5ba9e343b6f47c1846b14ebad56f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jan 2024 18:36:52 +0000 Subject: [PATCH 03/12] chore(deps): bump seaborn from 0.13.1 to 0.13.2 Bumps [seaborn](https://github.com/mwaskom/seaborn) from 0.13.1 to 0.13.2. - [Release notes](https://github.com/mwaskom/seaborn/releases) - [Commits](https://github.com/mwaskom/seaborn/compare/v0.13.1...v0.13.2) --- updated-dependencies: - dependency-name: seaborn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5126983d..b6c8e087 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ Pillow==10.2.0 pytest==7.4.4 scikit_learn==1.4.0 scipy==1.12.0 -seaborn==0.13.1 +seaborn==0.13.2 sparse==0.15.1 tqdm==4.66.1 umap_learn==0.5.5 From 72a17f0c005f2fc3c4912978d71b52277e2d4074 Mon Sep 17 00:00:00 2001 From: RichieHakim Date: Thu, 25 Jan 2024 15:19:02 -0500 Subject: [PATCH 04/12] add macos-12 to build.yml and update opencv to <=4.9 --- .github/workflows/build.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 83954a85..f4829296 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,7 +36,7 @@ jobs: windows-2022, windows-2019, # # macos-latest, - # macos-12.0, + macos-12, macos-11.0, # macos-10.15, ] diff --git a/requirements.txt b/requirements.txt index 7ae9c728..702444c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ kymatio==0.3.0 matplotlib==3.8.2 natsort==8.4.0 numpy==1.26.3 -opencv_contrib_python==4.8.1.78 +opencv_contrib_python<=4.9.0.80 optuna==3.4.0 Pillow==10.2.0 pytest==7.4.4 From 8cb8e677e2ec343d49c4ffbbcfd4ff2f566bd046 Mon Sep 17 00:00:00 2001 From: RichieHakim Date: Thu, 25 Jan 2024 17:05:54 -0500 Subject: [PATCH 05/12] Add platform-specific dependency for macOS --- setup.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 8f2b5d98..947c8aff 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,14 @@ ## setup.py file for roicat from pathlib import Path +import copy +import platform from distutils.core import setup -import copy +## Get the parent directory of this file dir_parent = Path(__file__).parent +## Get requirements from requirements.txt def read_requirements(): with open(str(dir_parent / "requirements.txt"), "r") as req: content = req.read() ## read the file @@ -19,9 +22,9 @@ def read_requirements(): requirements = [req.replace(",", "").replace("\"", "").replace("\'", "").strip() for req in requirements] return requirements - deps_all = read_requirements() + ## Dependencies: latest versions of requirements ### remove everything starting and after the first =,>,<,! sign deps_names = [req.split('=')[0].split('>')[0].split('<')[0].split('!')[0] for req in deps_all] @@ -29,6 +32,15 @@ def read_requirements(): deps_all_latest = copy.deepcopy(deps_names) + +## Operating system specific dependencies +### OpenCV >= 4.9 is not supported on macOS < 12 +system, version = platform.system(), platform.mac_ver()[0] +if system == "Darwin" and version and ('opencv_contrib_python' in deps_all_dict): + if tuple(map(int, version.split('.'))) < (12, 0, 0): + deps_all_dict['opencv_contrib_python'] = "opencv_contrib_python<=4.8.1.78" + + ## Make different versions of dependencies ### Also pull out the version number from the requirements (specified in deps_all_dict values). deps_core = [deps_all_dict[dep] for dep in [ @@ -72,11 +84,12 @@ def read_requirements(): 'kymatio', ]] + deps_core + ## Get README.md with open(str(dir_parent / "README.md"), "r") as f: readme = f.read() -## Get version number +## Get ROICaT version number with open(str(dir_parent / "roicat" / "__init__.py"), "r") as f: for line in f: if line.startswith("__version__"): From 71847198731093060e4b572fac37c30473f0a801 Mon Sep 17 00:00:00 2001 From: RichieHakim Date: Thu, 25 Jan 2024 17:13:20 -0500 Subject: [PATCH 06/12] bug fix --- setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 947c8aff..2bb2b5ad 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def read_requirements(): deps_names = [req.split('=')[0].split('>')[0].split('<')[0].split('!')[0] for req in deps_all] deps_all_dict = dict(zip(deps_names, deps_all)) -deps_all_latest = copy.deepcopy(deps_names) +deps_all_latest = dict(zip(deps_names, deps_names)) ## Operating system specific dependencies @@ -38,7 +38,8 @@ def read_requirements(): system, version = platform.system(), platform.mac_ver()[0] if system == "Darwin" and version and ('opencv_contrib_python' in deps_all_dict): if tuple(map(int, version.split('.'))) < (12, 0, 0): - deps_all_dict['opencv_contrib_python'] = "opencv_contrib_python<=4.8.1.78" + version_opencv_macosSub12 = "opencv_contrib_python<=4.8.1.78" + deps_all_dict['opencv_contrib_python'], deps_all_latest['opencv_contrib_python'] = version_opencv_macosSub12, version_opencv_macosSub12 ## Make different versions of dependencies @@ -65,21 +66,20 @@ def read_requirements(): 'psutil', 'py_cpuinfo', 'GPUtil', + 'skl2onnx', + 'onnx', + 'onnxruntime', + 'opencv_contrib_python', ]] deps_classification = [deps_all_dict[dep] for dep in [ - 'opencv_contrib_python', 'umap_learn', 'bokeh', 'holoviews[recommended]', 'jupyter_bokeh', - 'skl2onnx', - 'onnx', - 'onnxruntime', ]] + deps_core deps_tracking = [deps_all_dict[dep] for dep in [ - 'opencv_contrib_python', 'hdbscan', 'kymatio', ]] + deps_core From 893cb0e246b83f4e1187086adca34e4365bbecfa Mon Sep 17 00:00:00 2001 From: RichieHakim Date: Thu, 25 Jan 2024 17:51:19 -0500 Subject: [PATCH 07/12] Refactor OpenCV version for macOS < 12 --- setup.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 2bb2b5ad..96137958 100644 --- a/setup.py +++ b/setup.py @@ -33,13 +33,20 @@ def read_requirements(): deps_all_latest = dict(zip(deps_names, deps_names)) -## Operating system specific dependencies -### OpenCV >= 4.9 is not supported on macOS < 12 -system, version = platform.system(), platform.mac_ver()[0] -if system == "Darwin" and version and ('opencv_contrib_python' in deps_all_dict): - if tuple(map(int, version.split('.'))) < (12, 0, 0): - version_opencv_macosSub12 = "opencv_contrib_python<=4.8.1.78" - deps_all_dict['opencv_contrib_python'], deps_all_latest['opencv_contrib_python'] = version_opencv_macosSub12, version_opencv_macosSub12 +# Operating system specific dependencies +# OpenCV >= 4.9 is not supported on macOS < 12 +system, version_macos = platform.system(), platform.mac_ver()[0] + +if system == "Darwin" and version_macos: + # Safely convert version string components to integers + version_parts = version_macos.split('.') + version_major_macos= int(version_parts[0]) + + # Check macOS version and adjust the OpenCV version accordingly + if (version_major_macos < 12) and ('opencv_contrib_python' in deps_all_dict): + version_opencv_macos_sub12 = "opencv_contrib_python<=4.8.1.78" + deps_all_dict['opencv_contrib_python'] = version_opencv_macos_sub12 + deps_all_latest['opencv_contrib_python'] = version_opencv_macos_sub12 ## Make different versions of dependencies From 7b82b3bc2da75fa27b6ec34d62a41ed1c8414ac4 Mon Sep 17 00:00:00 2001 From: RichieHakim Date: Thu, 25 Jan 2024 20:13:44 -0500 Subject: [PATCH 08/12] trying to debug why opencv stuff isn't working --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 96137958..348bce7c 100644 --- a/setup.py +++ b/setup.py @@ -36,15 +36,16 @@ def read_requirements(): # Operating system specific dependencies # OpenCV >= 4.9 is not supported on macOS < 12 system, version_macos = platform.system(), platform.mac_ver()[0] - -if system == "Darwin" and version_macos: +print(f"System: {system}, macOS version: {version_macos}") +if (system == "Darwin"): # Safely convert version string components to integers version_parts = version_macos.split('.') - version_major_macos= int(version_parts[0]) + version_major_macos = int(version_parts[0]) # Check macOS version and adjust the OpenCV version accordingly if (version_major_macos < 12) and ('opencv_contrib_python' in deps_all_dict): version_opencv_macos_sub12 = "opencv_contrib_python<=4.8.1.78" + print(f"Detected macOS version {version_major_macos}, which is < 12. Installing an older version of OpenCV: {version_opencv_macos_sub12}") deps_all_dict['opencv_contrib_python'] = version_opencv_macos_sub12 deps_all_latest['opencv_contrib_python'] = version_opencv_macos_sub12 From d72fe88b6bdca6b5f43ab7233baea6af09b64c71 Mon Sep 17 00:00:00 2001 From: RichieHakim Date: Thu, 25 Jan 2024 20:19:51 -0500 Subject: [PATCH 09/12] bug fix for install --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 348bce7c..0726423b 100644 --- a/setup.py +++ b/setup.py @@ -124,8 +124,8 @@ def read_requirements(): install_requires=[], extras_require={ - 'all': deps_all, - 'all_latest': deps_all_latest, + 'all': list(deps_all_dict.values()), + 'all_latest': list(deps_all_latest.values()), 'core': deps_core, 'classification': deps_classification, 'tracking': deps_tracking, From 41450a984b5cc330e7ac9eb1d251d6e2ca138c17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jan 2024 18:33:35 +0000 Subject: [PATCH 10/12] chore(deps): bump bokeh from 3.3.3 to 3.3.4 Bumps [bokeh](https://github.com/bokeh/bokeh) from 3.3.3 to 3.3.4. - [Changelog](https://github.com/bokeh/bokeh/blob/3.3.4/docs/CHANGELOG) - [Commits](https://github.com/bokeh/bokeh/compare/3.3.3...3.3.4) --- updated-dependencies: - dependency-name: bokeh dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c1e142c5..14812df2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ sparse==0.15.1 tqdm==4.66.1 umap_learn==0.5.5 xxhash==3.4.1 -bokeh==3.3.3 +bokeh==3.3.4 psutil==5.9.8 py_cpuinfo==9.0.0 GPUtil==1.4.0 From 1c8f06a624818d5fd2be7d5ba225053fea4727b4 Mon Sep 17 00:00:00 2001 From: RichieHakim Date: Sun, 28 Jan 2024 13:20:44 -0500 Subject: [PATCH 11/12] Refactor parameter dictionary helpers into helpers --- roicat/helpers.py | 82 +++++++++++++++++++++++++++++++++++++++ roicat/pipelines.py | 4 +- roicat/util.py | 78 ------------------------------------- tests/test_integration.py | 2 +- 4 files changed, 85 insertions(+), 81 deletions(-) diff --git a/roicat/helpers.py b/roicat/helpers.py index 376006f1..36acee14 100644 --- a/roicat/helpers.py +++ b/roicat/helpers.py @@ -200,6 +200,88 @@ def flatten_dict(d: MutableMapping, parent_key: str = '', sep: str ='.') -> Muta items.append((new_key, v)) return dict(items) +## parameter dictionary helpers ## + +def fill_in_dict( + d: Dict, + defaults: Dict, + verbose: bool = True, + hierarchy: List[str] = ['dict'], +): + """ + In-place. Fills in dictionary ``d`` with values from ``defaults`` if they + are missing. Works hierachically. + RH 2023 + + Args: + d (Dict): + Dictionary to fill in. + In-place. + defaults (Dict): + Dictionary of defaults. + verbose (bool): + Whether to print messages. + hierarchy (List[str]): + Used internally for recursion. + Hierarchy of keys to d. + """ + from copy import deepcopy + for key in defaults: + if key not in d: + print(f"Key '{key}' not found in params dictionary: {' > '.join([f'{str(h)}' for h in hierarchy])}. Using default value: {defaults[key]}") if verbose else None + d.update({key: deepcopy(defaults[key])}) + elif isinstance(defaults[key], dict): + assert isinstance(d[key], dict), f"Key '{key}' is a dict in defaults, but not in params. {' > '.join([f'{str(h)}' for h in hierarchy])}." + fill_in_dict(d[key], defaults[key], hierarchy=hierarchy+[key]) + + +def check_keys_subset(d, default_dict, hierarchy=['defaults']): + """ + Checks that the keys in d are all in default_dict. Raises an error if not. + RH 2023 + + Args: + d (Dict): + Dictionary to check. + default_dict (Dict): + Dictionary containing the keys to check against. + hierarchy (List[str]): + Used internally for recursion. + Hierarchy of keys to d. + """ + default_keys = list(default_dict.keys()) + for key in d.keys(): + assert key in default_keys, f"Parameter '{key}' not found in defaults dictionary: {' > '.join([f'{str(h)}' for h in hierarchy])}." + if isinstance(default_dict[key], dict) and isinstance(d[key], dict): + check_keys_subset(d[key], default_dict[key], hierarchy=hierarchy+[key]) + + +def prepare_params(params, defaults, verbose=True): + """ + Does the following: + * Checks that all keys in ``params`` are in ``defaults``. + * Fills in any missing keys in ``params`` with values from ``defaults``. + * Returns a deepcopy of the filled-in ``params``. + + Args: + params (Dict): + Dictionary of parameters. + defaults (Dict): + Dictionary of defaults. + verbose (bool): + Whether to print messages. + """ + from copy import deepcopy + ## Check inputs + assert isinstance(params, dict), f"p must be a dict. Got {type(params)} instead." + ## Make sure all the keys in p are valid + check_keys_subset(params, defaults) + ## Fill in any missing keys with defaults + params_out = deepcopy(params) + fill_in_dict(params_out, defaults, verbose=verbose) + + return params_out + ###################################################################################################################################### ####################################################### MATH FUNCTIONS ############################################################### diff --git a/roicat/pipelines.py b/roicat/pipelines.py index 75d71074..f351c6fa 100644 --- a/roicat/pipelines.py +++ b/roicat/pipelines.py @@ -30,12 +30,12 @@ def pipeline_tracking(params: dict): pipeline. params (dict): Parameters used in the pipeline. See - ``roicat.util.prepare_params()`` for details. + ``roicat.helpers.prepare_params()`` for details. """ ## Prepare params defaults = util.get_default_parameters(pipeline='tracking') - params = util.prepare_params(params, defaults, verbose=True) + params = helpers.prepare_params(params, defaults, verbose=True) display(params) ## Prepare state variables diff --git a/roicat/util.py b/roicat/util.py index 30e49049..e0fedbe2 100644 --- a/roicat/util.py +++ b/roicat/util.py @@ -250,84 +250,6 @@ def get_default_parameters( return out -def fill_in_params( - d: Dict, - defaults: Dict, - verbose: bool = True, - hierarchy: List[str] = ['params'], -): - """ - In-place. Fills in dictionary ``d`` with values from ``defaults`` if they - are missing. Works hierachically. - RH 2023 - - Args: - d (Dict): - Dictionary to fill in. - In-place. - defaults (Dict): - Dictionary of defaults. - verbose (bool): - Whether to print messages. - hierarchy (List[str]): - Used internally for recursion. - Hierarchy of keys to d. - """ - from copy import deepcopy - for key in defaults: - if key not in d: - print(f"Parameter '{key}' not found in params dictionary: {' > '.join([f'{str(h)}' for h in hierarchy])}. Using default value: {defaults[key]}") if verbose else None - d.update({key: deepcopy(defaults[key])}) - elif isinstance(defaults[key], dict): - assert isinstance(d[key], dict), f"Parameter '{key}' must be a dictionary." - fill_in_params(d[key], defaults[key], hierarchy=hierarchy+[key]) - -def check_keys_subset(d, default_dict, hierarchy=['defaults']): - """ - Checks that the keys in d are all in default_dict. Raises an error if not. - RH 2023 - - Args: - d (Dict): - Dictionary to check. - default_dict (Dict): - Dictionary containing the keys to check against. - hierarchy (List[str]): - Used internally for recursion. - Hierarchy of keys to d. - """ - default_keys = list(default_dict.keys()) - for key in d.keys(): - assert key in default_keys, f"Parameter '{key}' not found in defaults dictionary: {' > '.join([f'{str(h)}' for h in hierarchy])}." - if isinstance(default_dict[key], dict) and isinstance(d[key], dict): - check_keys_subset(d[key], default_dict[key], hierarchy=hierarchy+[key]) - -def prepare_params(params, defaults, verbose=True): - """ - Does the following: - * Checks that all keys in ``params`` are in ``defaults``. - * Fills in any missing keys in ``params`` with values from ``defaults``. - * Returns a deepcopy of the filled-in ``params``. - - Args: - params (Dict): - Dictionary of parameters. - defaults (Dict): - Dictionary of defaults. - verbose (bool): - Whether to print messages. - """ - from copy import deepcopy - ## Check inputs - assert isinstance(params, dict), f"p must be a dict. Got {type(params)} instead." - ## Make sure all the keys in p are valid - check_keys_subset(params, defaults) - ## Fill in any missing keys with defaults - params_out = deepcopy(params) - fill_in_params(params_out, defaults, verbose=verbose) - - return params_out - def system_info(verbose: bool = False,) -> Dict: """ diff --git a/tests/test_integration.py b/tests/test_integration.py index 6688d65b..e0fb9bcb 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -62,7 +62,7 @@ def test_pipeline_tracking_simple(dir_data_test): }, }, } - params = util.prepare_params(params_partial, defaults) + params = helpers.prepare_params(params_partial, defaults) results, run_data, params = pipelines.pipeline_tracking(params) #check that results fields are not empty From 3929c447841f0dc770554f730392c213edd6fec8 Mon Sep 17 00:00:00 2001 From: RichieHakim Date: Sun, 28 Jan 2024 13:25:39 -0500 Subject: [PATCH 12/12] Bumped up version to 1.1.29 --- roicat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roicat/__init__.py b/roicat/__init__.py index 9ea1e53c..150c539e 100644 --- a/roicat/__init__.py +++ b/roicat/__init__.py @@ -13,4 +13,4 @@ for pkg in __all__: exec('from . import ' + pkg) -__version__ = '1.1.28' +__version__ = '1.1.29'