From d6fcaa9526654918d909855dd906748407ce6b38 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 7 Sep 2023 12:17:19 -0700 Subject: [PATCH 01/18] added TODO --- tests/test_docstrings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 26588dc3..1c022584 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -51,7 +51,7 @@ def test_has_docstring(obj): msg = f"{obj.__name__} has no docstring." else: msg = f"{obj.__module__}.{obj.__qualname__} has no docstring." - if "__create_fn__" in msg: + if "__create_fn__" in msg: # TODO: add skip for Enums return # skip dataclass functions created by __create_fn__ assert doc is not None, msg From df509ce9ca7096853add7850bcd53b54a62b9a03 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Fri, 29 Sep 2023 13:28:36 -0700 Subject: [PATCH 02/18] added Cody's comparison script --- tests/compare_docstring_testers.py | 105 +++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 tests/compare_docstring_testers.py diff --git a/tests/compare_docstring_testers.py b/tests/compare_docstring_testers.py new file mode 100644 index 00000000..21f6c362 --- /dev/null +++ b/tests/compare_docstring_testers.py @@ -0,0 +1,105 @@ +import inspect +import os +import importlib +from pathlib import Path +from types import ModuleType, FunctionType +from typing import List, Iterable + +import nwbinspector + + +def traverse_class(cls, objs): + """Traverse a class and its methods and append them to objs.""" + predicate = lambda x: inspect.isfunction(x) or inspect.ismethod(x) + for name, obj in inspect.getmembers(cls, predicate=predicate): + objs.append(obj) + + +def traverse_module(module, objs): + """Traverse all classes and functions in a module and append them to objs.""" + objs.append(module) + predicate = lambda x: inspect.isclass(x) or inspect.isfunction(x) or inspect.ismethod(x) + for name, obj in inspect.getmembers(module, predicate=predicate): + parent_package = obj.__module__.split(".")[0] + if parent_package != "nwbinspector": # avoid traversing external dependencies + continue + objs.append(obj) + if inspect.isclass(obj): + traverse_class(obj, objs) + + +def traverse_package(package, objs): + """Traverse all modules and subpackages in a package to append all members to objs.""" + for child in os.listdir(package.__path__[0]): + if child.startswith(".") or child == "__pycache__": + continue + elif child.endswith(".py"): + module_name = child[:-3] + module = importlib.import_module(f"{package.__name__}.{module_name}") + traverse_module(module, objs) + elif Path(child).is_dir(): # subpackage - I did change this one line b/c error otherwise when hit a .json + subpackage = importlib.import_module(f"{package.__name__}.{child}") + traverse_package(subpackage, objs) + + +def traverse_class_2(class_object: type, parent: str) -> List[FunctionType]: + """Traverse the class dictionary and return the methods overridden by this module.""" + class_functions = list() + for attribute_name, attribute_value in class_object.__dict__.items(): + if isinstance(attribute_value, FunctionType) and attribute_value.__module__.startswith(parent): + class_functions.append(attribute_value) + return class_functions + + +def traverse_module_2(module: ModuleType, parent: str) -> Iterable[FunctionType]: + """Traverse the module directory and return all submodules, classes, and functions defined along the way.""" + local_modules_classes_and_functions = list() + + for name in dir(module): + if name.startswith("__") and name.endswith("__"): # skip all magic methods + continue + + object_ = getattr(module, name) + + if isinstance(object_, ModuleType) and object_.__package__.startswith(parent): + submodule = object_ + + submodule_functions = traverse_module_2(module=submodule, parent=parent) + + local_modules_classes_and_functions.append(submodule) + local_modules_classes_and_functions.extend(submodule_functions) + elif isinstance(object_, type) and object_.__module__.startswith(parent): # class + class_object = object_ + + class_functions = traverse_class_2(class_object=class_object, parent=parent) + + local_modules_classes_and_functions.append(class_object) + local_modules_classes_and_functions.extend(class_functions) + elif isinstance(object_, FunctionType) and object_.__module__.startswith(parent): + function = object_ + + local_modules_classes_and_functions.append(function) + + return local_modules_classes_and_functions + + +list_1 = list() +traverse_package(package=nwbinspector, objs=list_1) + +list_2 = traverse_module_2(module=nwbinspector, parent="nwbinspector") + +# Analyze and compare - note that for set comparison, the lists must have been run in the same kernel +# to give all imports the same address in memory +unique_list_1 = set(list_1) +unique_list_2 = set(list_2) + +found_by_2_and_not_by_1 = unique_list_2 - unique_list_1 + +# Summary: A series of nested submodules under `checks` and `tools`; some various private functions scattered around +# not really clear why Paul's missed these + +found_by_1_and_not_by_2 = unique_list_1 - unique_list_2 + +# Summary: All of these are bound methods of the Enum's (Importance/Severity) or JSONEncoder +# and are not methods that we actually override in the codebase (they strictly inherit) +# It did, however, find the outermost package __init__ (does that really need a docstring though?) From f7bfa10d3e4e0f3b34315460fb5eb00fbaa90f25 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Fri, 29 Sep 2023 16:49:08 -0700 Subject: [PATCH 03/18] revamped test_docstrings to use Cody's cleaner implementation + minor tweaks --- tests/compare_docstring_testers.py | 98 +++++++++++++++++++++- tests/test_docstrings.py | 129 ++++++++++++++++++++++------- 2 files changed, 192 insertions(+), 35 deletions(-) diff --git a/tests/compare_docstring_testers.py b/tests/compare_docstring_testers.py index 21f6c362..d1db755b 100644 --- a/tests/compare_docstring_testers.py +++ b/tests/compare_docstring_testers.py @@ -4,8 +4,9 @@ from pathlib import Path from types import ModuleType, FunctionType from typing import List, Iterable +from pprint import pprint -import nwbinspector +import roiextractors def traverse_class(cls, objs): @@ -21,7 +22,7 @@ def traverse_module(module, objs): predicate = lambda x: inspect.isclass(x) or inspect.isfunction(x) or inspect.ismethod(x) for name, obj in inspect.getmembers(module, predicate=predicate): parent_package = obj.__module__.split(".")[0] - if parent_package != "nwbinspector": # avoid traversing external dependencies + if parent_package != "roiextractors": # avoid traversing external dependencies continue objs.append(obj) if inspect.isclass(obj): @@ -83,23 +84,112 @@ def traverse_module_2(module: ModuleType, parent: str) -> Iterable[FunctionType] return local_modules_classes_and_functions +def traverse_class3(class_object: type, parent: str) -> List[FunctionType]: + """Traverse the class dictionary and return the methods overridden by this module.""" + class_functions = [] + for attribute_name, attribute_value in class_object.__dict__.items(): + if isinstance(attribute_value, FunctionType) and attribute_value.__module__.startswith(parent): + if attribute_name.startswith("__") and attribute_name.endswith("__"): + continue + class_functions.append(attribute_value) + return class_functions + + +def traverse_module3(module: ModuleType, parent: str) -> List: + local_classes_and_functions = [] + + for name in dir(module): + if name.startswith("__") and name.endswith("__"): # skip all magic methods + continue + + object_ = getattr(module, name) + + if isinstance(object_, type) and object_.__module__.startswith(parent): # class + class_object = object_ + class_functions = traverse_class3(class_object=class_object, parent=parent) + local_classes_and_functions.append(class_object) + local_classes_and_functions.extend(class_functions) + + elif isinstance(object_, FunctionType) and object_.__module__.startswith(parent): + function = object_ + local_classes_and_functions.append(function) + + return local_classes_and_functions + + +def traverse_package3(package: ModuleType, parent: str) -> List[ModuleType]: + """Traverse the package and return all subpackages and modules defined along the way. + + Parameters + ---------- + package : ModuleType + The package, subpackage, or module to traverse. + parent : str + The parent package name. + + Returns + ------- + local_packages_and_modules : List[ModuleType] + A list of all subpackages and modules defined in the given package. + """ + local_packages_and_modules = [] + + for name in dir(package): + if name.startswith("__") and name.endswith("__"): # skip all magic methods + continue + + object_ = getattr(package, name) + + if ( + isinstance(object_, ModuleType) + and object_.__file__[-11:] == "__init__.py" + and object_.__package__.startswith(parent) + ): + subpackage = object_ + subpackage_members = traverse_package3(package=subpackage, parent=parent) + local_packages_and_modules.append(subpackage) + local_packages_and_modules.extend(subpackage_members) + + elif isinstance(object_, ModuleType) and object_.__package__.startswith(parent): + module = object_ + module_members = traverse_module3(module=module, parent=parent) + local_packages_and_modules.append(module) + local_packages_and_modules.extend(module_members) + + return local_packages_and_modules + + list_1 = list() -traverse_package(package=nwbinspector, objs=list_1) +traverse_package(package=roiextractors, objs=list_1) -list_2 = traverse_module_2(module=nwbinspector, parent="nwbinspector") +list_2 = traverse_module_2(module=roiextractors, parent="roiextractors") +list_3 = traverse_package3(package=roiextractors, parent="roiextractors") # Analyze and compare - note that for set comparison, the lists must have been run in the same kernel # to give all imports the same address in memory unique_list_1 = set(list_1) unique_list_2 = set(list_2) +unique_list_3 = set(list_3) found_by_2_and_not_by_1 = unique_list_2 - unique_list_1 +print("found by 2 and not by 1:") +pprint(found_by_2_and_not_by_1) # Summary: A series of nested submodules under `checks` and `tools`; some various private functions scattered around # not really clear why Paul's missed these found_by_1_and_not_by_2 = unique_list_1 - unique_list_2 +print("found by 1 and not by 2:") +pprint(found_by_1_and_not_by_2) # Summary: All of these are bound methods of the Enum's (Importance/Severity) or JSONEncoder # and are not methods that we actually override in the codebase (they strictly inherit) # It did, however, find the outermost package __init__ (does that really need a docstring though?) + +found_by_3_and_not_by_2 = unique_list_3 - unique_list_2 +print("found by 3 and not by 2:") +pprint(found_by_3_and_not_by_2) + +found_by_2_and_not_by_3 = unique_list_2 - unique_list_3 +print("found by 2 and not by 3:") +pprint(found_by_2_and_not_by_3) diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 1c022584..344f513e 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -1,46 +1,115 @@ import inspect import os import importlib -import roiextractors +from types import ModuleType, FunctionType +from typing import List, Iterable import pytest +import roiextractors + + +def traverse_class(class_object: type, parent: str) -> List[FunctionType]: + """Traverse the class dictionary and return the methods overridden by this module. + + Parameters + ---------- + class_object : type + The class to traverse. + parent : str + The parent package name. + Returns + ------- + class_methods : List[FunctionType] + A list of all methods defined in the given class. + """ + class_methods = [] + for attribute_name, attribute_value in class_object.__dict__.items(): + if isinstance(attribute_value, FunctionType) and attribute_value.__module__.startswith(parent): + if attribute_name.startswith("__") and attribute_name.endswith("__"): + continue + class_methods.append(attribute_value) + return class_methods -def traverse_class(cls, objs): - """Traverse a class and its methods and append them to objs.""" - predicate = lambda x: inspect.isfunction(x) or inspect.ismethod(x) - for name, obj in inspect.getmembers(cls, predicate=predicate): - objs.append(obj) +def traverse_module(module: ModuleType, parent: str) -> List: + """Traverse the module and return all classes and functions defined along the way. -def traverse_module(module, objs): - """Traverse all classes and functions in a module and append them to objs.""" - objs.append(module) - predicate = lambda x: inspect.isclass(x) or inspect.isfunction(x) or inspect.ismethod(x) - for name, obj in inspect.getmembers(module, predicate=predicate): - parent_package = obj.__module__.split(".")[0] - if parent_package != "roiextractors": # avoid traversing external dependencies + Parameters + ---------- + module : ModuleType + The module to traverse. + parent : str + The parent package name. + + Returns + ------- + local_classes_and_functions : List + A list of all classes and functions defined in the given module. + """ + local_classes_and_functions = [] + + for name in dir(module): + if name.startswith("__") and name.endswith("__"): # skip all magic methods continue - objs.append(obj) - if inspect.isclass(obj): - traverse_class(obj, objs) + object_ = getattr(module, name) + + if isinstance(object_, type) and object_.__module__.startswith(parent): # class + class_object = object_ + class_functions = traverse_class(class_object=class_object, parent=parent) + local_classes_and_functions.append(class_object) + local_classes_and_functions.extend(class_functions) + + elif isinstance(object_, FunctionType) and object_.__module__.startswith(parent): + function = object_ + local_classes_and_functions.append(function) -def traverse_package(package, objs): - """Traverse all modules and subpackages in a package to append all members to objs.""" - for child in os.listdir(package.__path__[0]): - if child.startswith(".") or child == "__pycache__": + return local_classes_and_functions + + +def traverse_package(package: ModuleType, parent: str) -> List[ModuleType]: + """Traverse the package and return all subpackages and modules defined along the way. + + Parameters + ---------- + package : ModuleType + The package, subpackage, or module to traverse. + parent : str + The parent package name. + + Returns + ------- + local_packages_and_modules : List[ModuleType] + A list of all subpackages and modules defined in the given package. + """ + local_packages_and_modules = [] + + for name in dir(package): + if name.startswith("__") and name.endswith("__"): # skip all magic methods continue - elif child.endswith(".py"): - module_name = child[:-3] - module = importlib.import_module(f"{package.__name__}.{module_name}") - traverse_module(module, objs) - else: # subpackage - subpackage = importlib.import_module(f"{package.__name__}.{child}") - traverse_package(subpackage, objs) + + object_ = getattr(package, name) + + if ( + isinstance(object_, ModuleType) + and object_.__file__[-11:] == "__init__.py" + and object_.__package__.startswith(parent) + ): + subpackage = object_ + subpackage_members = traverse_package(package=subpackage, parent=parent) + local_packages_and_modules.append(subpackage) + local_packages_and_modules.extend(subpackage_members) + + elif isinstance(object_, ModuleType) and object_.__package__.startswith(parent): + module = object_ + module_members = traverse_module(module=module, parent=parent) + local_packages_and_modules.append(module) + local_packages_and_modules.extend(module_members) + + return local_packages_and_modules -objs = [] -traverse_package(roiextractors, objs) +objs = traverse_package(roiextractors, "roiextractors") @pytest.mark.parametrize("obj", objs) @@ -51,8 +120,6 @@ def test_has_docstring(obj): msg = f"{obj.__name__} has no docstring." else: msg = f"{obj.__module__}.{obj.__qualname__} has no docstring." - if "__create_fn__" in msg: # TODO: add skip for Enums - return # skip dataclass functions created by __create_fn__ assert doc is not None, msg From 1afea15492da6f6a7929784f76fb3398917dc294 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 19 Oct 2023 09:32:57 -0700 Subject: [PATCH 04/18] removed docstring comparison script --- tests/compare_docstring_testers.py | 195 ----------------------------- 1 file changed, 195 deletions(-) delete mode 100644 tests/compare_docstring_testers.py diff --git a/tests/compare_docstring_testers.py b/tests/compare_docstring_testers.py deleted file mode 100644 index d1db755b..00000000 --- a/tests/compare_docstring_testers.py +++ /dev/null @@ -1,195 +0,0 @@ -import inspect -import os -import importlib -from pathlib import Path -from types import ModuleType, FunctionType -from typing import List, Iterable -from pprint import pprint - -import roiextractors - - -def traverse_class(cls, objs): - """Traverse a class and its methods and append them to objs.""" - predicate = lambda x: inspect.isfunction(x) or inspect.ismethod(x) - for name, obj in inspect.getmembers(cls, predicate=predicate): - objs.append(obj) - - -def traverse_module(module, objs): - """Traverse all classes and functions in a module and append them to objs.""" - objs.append(module) - predicate = lambda x: inspect.isclass(x) or inspect.isfunction(x) or inspect.ismethod(x) - for name, obj in inspect.getmembers(module, predicate=predicate): - parent_package = obj.__module__.split(".")[0] - if parent_package != "roiextractors": # avoid traversing external dependencies - continue - objs.append(obj) - if inspect.isclass(obj): - traverse_class(obj, objs) - - -def traverse_package(package, objs): - """Traverse all modules and subpackages in a package to append all members to objs.""" - for child in os.listdir(package.__path__[0]): - if child.startswith(".") or child == "__pycache__": - continue - elif child.endswith(".py"): - module_name = child[:-3] - module = importlib.import_module(f"{package.__name__}.{module_name}") - traverse_module(module, objs) - elif Path(child).is_dir(): # subpackage - I did change this one line b/c error otherwise when hit a .json - subpackage = importlib.import_module(f"{package.__name__}.{child}") - traverse_package(subpackage, objs) - - -def traverse_class_2(class_object: type, parent: str) -> List[FunctionType]: - """Traverse the class dictionary and return the methods overridden by this module.""" - class_functions = list() - for attribute_name, attribute_value in class_object.__dict__.items(): - if isinstance(attribute_value, FunctionType) and attribute_value.__module__.startswith(parent): - class_functions.append(attribute_value) - return class_functions - - -def traverse_module_2(module: ModuleType, parent: str) -> Iterable[FunctionType]: - """Traverse the module directory and return all submodules, classes, and functions defined along the way.""" - local_modules_classes_and_functions = list() - - for name in dir(module): - if name.startswith("__") and name.endswith("__"): # skip all magic methods - continue - - object_ = getattr(module, name) - - if isinstance(object_, ModuleType) and object_.__package__.startswith(parent): - submodule = object_ - - submodule_functions = traverse_module_2(module=submodule, parent=parent) - - local_modules_classes_and_functions.append(submodule) - local_modules_classes_and_functions.extend(submodule_functions) - elif isinstance(object_, type) and object_.__module__.startswith(parent): # class - class_object = object_ - - class_functions = traverse_class_2(class_object=class_object, parent=parent) - - local_modules_classes_and_functions.append(class_object) - local_modules_classes_and_functions.extend(class_functions) - elif isinstance(object_, FunctionType) and object_.__module__.startswith(parent): - function = object_ - - local_modules_classes_and_functions.append(function) - - return local_modules_classes_and_functions - - -def traverse_class3(class_object: type, parent: str) -> List[FunctionType]: - """Traverse the class dictionary and return the methods overridden by this module.""" - class_functions = [] - for attribute_name, attribute_value in class_object.__dict__.items(): - if isinstance(attribute_value, FunctionType) and attribute_value.__module__.startswith(parent): - if attribute_name.startswith("__") and attribute_name.endswith("__"): - continue - class_functions.append(attribute_value) - return class_functions - - -def traverse_module3(module: ModuleType, parent: str) -> List: - local_classes_and_functions = [] - - for name in dir(module): - if name.startswith("__") and name.endswith("__"): # skip all magic methods - continue - - object_ = getattr(module, name) - - if isinstance(object_, type) and object_.__module__.startswith(parent): # class - class_object = object_ - class_functions = traverse_class3(class_object=class_object, parent=parent) - local_classes_and_functions.append(class_object) - local_classes_and_functions.extend(class_functions) - - elif isinstance(object_, FunctionType) and object_.__module__.startswith(parent): - function = object_ - local_classes_and_functions.append(function) - - return local_classes_and_functions - - -def traverse_package3(package: ModuleType, parent: str) -> List[ModuleType]: - """Traverse the package and return all subpackages and modules defined along the way. - - Parameters - ---------- - package : ModuleType - The package, subpackage, or module to traverse. - parent : str - The parent package name. - - Returns - ------- - local_packages_and_modules : List[ModuleType] - A list of all subpackages and modules defined in the given package. - """ - local_packages_and_modules = [] - - for name in dir(package): - if name.startswith("__") and name.endswith("__"): # skip all magic methods - continue - - object_ = getattr(package, name) - - if ( - isinstance(object_, ModuleType) - and object_.__file__[-11:] == "__init__.py" - and object_.__package__.startswith(parent) - ): - subpackage = object_ - subpackage_members = traverse_package3(package=subpackage, parent=parent) - local_packages_and_modules.append(subpackage) - local_packages_and_modules.extend(subpackage_members) - - elif isinstance(object_, ModuleType) and object_.__package__.startswith(parent): - module = object_ - module_members = traverse_module3(module=module, parent=parent) - local_packages_and_modules.append(module) - local_packages_and_modules.extend(module_members) - - return local_packages_and_modules - - -list_1 = list() -traverse_package(package=roiextractors, objs=list_1) - -list_2 = traverse_module_2(module=roiextractors, parent="roiextractors") -list_3 = traverse_package3(package=roiextractors, parent="roiextractors") - -# Analyze and compare - note that for set comparison, the lists must have been run in the same kernel -# to give all imports the same address in memory -unique_list_1 = set(list_1) -unique_list_2 = set(list_2) -unique_list_3 = set(list_3) - -found_by_2_and_not_by_1 = unique_list_2 - unique_list_1 -print("found by 2 and not by 1:") -pprint(found_by_2_and_not_by_1) - -# Summary: A series of nested submodules under `checks` and `tools`; some various private functions scattered around -# not really clear why Paul's missed these - -found_by_1_and_not_by_2 = unique_list_1 - unique_list_2 -print("found by 1 and not by 2:") -pprint(found_by_1_and_not_by_2) - -# Summary: All of these are bound methods of the Enum's (Importance/Severity) or JSONEncoder -# and are not methods that we actually override in the codebase (they strictly inherit) -# It did, however, find the outermost package __init__ (does that really need a docstring though?) - -found_by_3_and_not_by_2 = unique_list_3 - unique_list_2 -print("found by 3 and not by 2:") -pprint(found_by_3_and_not_by_2) - -found_by_2_and_not_by_3 = unique_list_2 - unique_list_3 -print("found by 2 and not by 3:") -pprint(found_by_2_and_not_by_3) From 00f5af67f1bb8fcefda17e716bd4924957653731 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 08:03:51 +0000 Subject: [PATCH 05/18] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 23.7.0 → 23.9.1](https://github.com/psf/black/compare/23.7.0...23.9.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f77bafe..345fe75b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 23.9.1 hooks: - id: black exclude: ^docs/ From d878a6b560aea2ec189252ea4824d07e1e63c68c Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 11:24:07 -0700 Subject: [PATCH 06/18] Made run-tests workflow dependent on update-testing-data workflow --- .github/workflows/run-tests.yml | 40 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index b68e169f..276921f5 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -4,6 +4,10 @@ on: - cron: "0 0 * * *" # daily pull_request: workflow_dispatch: + workflow_run: + workflows: ["Update testing data"] + types: + - completed jobs: run: @@ -41,24 +45,24 @@ jobs: - - name: Get ophys_testing_data current head hash - id: ophys - run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" - - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - uses: actions/cache@v2 - id: cache-ophys-datasets - with: - path: ./ophys_testing_data - key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Install and configure AWS CLI - run: | - pip install awscli==1.25.27 - aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Download data from S3 - run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data + # - name: Get ophys_testing_data current head hash + # id: ophys + # run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + # - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + # uses: actions/cache@v2 + # id: cache-ophys-datasets + # with: + # path: ./ophys_testing_data + # key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + # - if: steps.cache-ophys-datasets.outputs.cache-hit == false + # name: Install and configure AWS CLI + # run: | + # pip install awscli==1.25.27 + # aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # - if: steps.cache-ophys-datasets.outputs.cache-hit == false + # name: Download data from S3 + # run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data From ea9bb38c2b08b42b42cad594ce9bcefca39dfe42 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 11:36:20 -0700 Subject: [PATCH 07/18] update awscli version --- .github/workflows/update-testing-data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index a2551302..6d327aea 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -38,7 +38,7 @@ jobs: - if: steps.cache-ophys-datasets.outputs.cache-hit == false name: Install and configure AWS CLI run: | - pip install awscli==1.25.27 + pip install awscli==1.29.56 aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: steps.cache-ophys-datasets.outputs.cache-hit == false From e7182d86fcd064b277c40c0da4f43ca0b3cd02a3 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 11:47:47 -0700 Subject: [PATCH 08/18] pin wheel version to avoid future dependency conflicts --- .github/workflows/update-testing-data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index 6d327aea..91342a0e 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -24,7 +24,7 @@ jobs: pip install -U pip git config --global user.email "CI@example.com" git config --global user.name "CI Almighty" - pip install wheel # needed for scanimage + pip install wheel==0.41.2 # needed for scanimage - name: Get ophys_testing_data current head hash id: ophys From c7fa284120fb018f0ce00f1002263391ac5baa20 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 12:03:07 -0700 Subject: [PATCH 09/18] updated awscli version in run-tests --- .github/workflows/run-tests.yml | 38 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 276921f5..ce2dc45f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -33,7 +33,7 @@ jobs: pip install pytest-xdist git config --global user.email "CI@example.com" git config --global user.name "CI Almighty" - pip install wheel # needed for scanimage + pip install wheel==0.41.2 # needed for scanimage - name: Install roiextractors with minimal requirements run: pip install .[test] @@ -45,24 +45,24 @@ jobs: - # - name: Get ophys_testing_data current head hash - # id: ophys - # run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" - # - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - # uses: actions/cache@v2 - # id: cache-ophys-datasets - # with: - # path: ./ophys_testing_data - # key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - # - if: steps.cache-ophys-datasets.outputs.cache-hit == false - # name: Install and configure AWS CLI - # run: | - # pip install awscli==1.25.27 - # aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # - if: steps.cache-ophys-datasets.outputs.cache-hit == false - # name: Download data from S3 - # run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data + - name: Get ophys_testing_data current head hash + id: ophys + run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + uses: actions/cache@v2 + id: cache-ophys-datasets + with: + path: ./ophys_testing_data + key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + - if: steps.cache-ophys-datasets.outputs.cache-hit == false + name: Install and configure AWS CLI + run: | + pip install awscli==1.29.56 + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - if: steps.cache-ophys-datasets.outputs.cache-hit == false + name: Download data from S3 + run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data From ad0133d398f6a16b48e5358a97585bdd547c45f9 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 13:05:23 -0700 Subject: [PATCH 10/18] removed dependent run of run-tests --- .github/workflows/run-tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ce2dc45f..5941f1c6 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -4,10 +4,6 @@ on: - cron: "0 0 * * *" # daily pull_request: workflow_dispatch: - workflow_run: - workflows: ["Update testing data"] - types: - - completed jobs: run: From fbdd78247375157096398fb5da43516e54fff982 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 13:33:01 -0700 Subject: [PATCH 11/18] commented out awscli commands only --- .github/workflows/run-tests.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 5941f1c6..ce0e6463 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -50,15 +50,15 @@ jobs: with: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Install and configure AWS CLI - run: | - pip install awscli==1.29.56 - aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Download data from S3 - run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data + # - if: steps.cache-ophys-datasets.outputs.cache-hit == false + # name: Install and configure AWS CLI + # run: | + # pip install awscli==1.29.56 + # aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # - if: steps.cache-ophys-datasets.outputs.cache-hit == false + # name: Download data from S3 + # run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data From f6d5e1d1eb27d39c23c92dd7039edbb965c9263c Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 13:55:36 -0700 Subject: [PATCH 12/18] refactored to trigger run-tests after update-testing-data --- .github/workflows/run-tests.yml | 19 ++++--------------- .github/workflows/update-testing-data.yml | 4 +++- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ce0e6463..0b26dcc0 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,9 +1,10 @@ name: Full Tests on: - schedule: - - cron: "0 0 * * *" # daily pull_request: workflow_dispatch: + workflow_run: + workflows: [update-testing-data] + types: [completed] jobs: run: @@ -33,14 +34,13 @@ jobs: - name: Install roiextractors with minimal requirements run: pip install .[test] + - name: Run minimal tests run: pytest tests/test_internals -n auto --dist loadscope - name: Test full installation (-e needed for codecov report) run: pip install -e .[full] - - - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" @@ -50,17 +50,6 @@ jobs: with: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - # - if: steps.cache-ophys-datasets.outputs.cache-hit == false - # name: Install and configure AWS CLI - # run: | - # pip install awscli==1.29.56 - # aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # - if: steps.cache-ophys-datasets.outputs.cache-hit == false - # name: Download data from S3 - # run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data - - - name: Run full pytest with coverage run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index 91342a0e..d012a203 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -1,5 +1,7 @@ -name: Update testing data +name: update-testing-data on: + schedule: + - cron: "0 0 * * *" # daily workflow_dispatch: jobs: From 04ecacc6780edf3459598573c65d140b76cb446e Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 14:28:28 -0700 Subject: [PATCH 13/18] added success/failure logic --- .github/workflows/run-tests.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0b26dcc0..bf17e653 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -7,7 +7,13 @@ on: types: [completed] jobs: - run: + on-failure: + name: Notify on failure + runs-on: ${{ matrix.os }} + if: ${{ github.event.workflow_run.conclusion == 'failure' }} + steps: + - run: echo 'The triggering workflow failed.' + on-success: name: Full tests on ${{ matrix.os }} with Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: From 45f59a2c239b479078173ff207284a8ef110098e Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Thu, 28 Sep 2023 18:27:54 -0400 Subject: [PATCH 14/18] return 0 for failing case Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- .github/workflows/run-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index bf17e653..59dd49df 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -12,7 +12,9 @@ jobs: runs-on: ${{ matrix.os }} if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - - run: echo 'The triggering workflow failed.' + - run: | + echo 'The triggering workflow failed.' + 0 on-success: name: Full tests on ${{ matrix.os }} with Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} From 8abb0977fd75b84c06692f4784511383d182b166 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 28 Sep 2023 15:30:23 -0700 Subject: [PATCH 15/18] fixed indentation --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 59dd49df..1723bbd9 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -13,8 +13,8 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - run: | - echo 'The triggering workflow failed.' - 0 + echo 'The triggering workflow failed.' + 0 on-success: name: Full tests on ${{ matrix.os }} with Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} From 6016b584c16586302ac5958074a7bdd2c2d51500 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Thu, 28 Sep 2023 12:34:43 -0400 Subject: [PATCH 16/18] Fixed broken badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8272d0e..31e54759 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![PyPI version](https://badge.fury.io/py/roiextractors.svg)](https://badge.fury.io/py/roiextractors) -![Full Tests](https://github.com/catalystneuro/roiextractors/actions/workflows/testing.yml/badge.svg) +![Full Tests](https://github.com/catalystneuro/roiextractors/actions/workflows/run-tests.yml/badge.svg) ![Auto-release](https://github.com/catalystneuro/roiextractors/actions/workflows/auto-publish.yml/badge.svg) [![codecov](https://codecov.io/github/catalystneuro/roiextractors/coverage.svg?branch=master)](https://codecov.io/github/catalystneuro/roiextractors?branch=master) [![documentation](https://readthedocs.org/projects/roiextractors/badge/?version=latest)](https://roiextractors.readthedocs.io/en/latest/) From 2cdf51b07ce8b47a09909f9dd45e9c403aa4087f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 20:37:12 +0000 Subject: [PATCH 17/18] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.4.0 → v4.5.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.4.0...v4.5.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 345fe75b..456112df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer From b67dbd45e4043d94abadb41d3447f643fcf6c544 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 19 Oct 2023 09:54:47 -0700 Subject: [PATCH 18/18] removed unnecessary imports --- tests/test_docstrings.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 344f513e..ff72ffd4 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -1,8 +1,6 @@ import inspect -import os -import importlib from types import ModuleType, FunctionType -from typing import List, Iterable +from typing import List import pytest import roiextractors