Merge branch 'main' into update_dl_doc

.gitattributes

@@ -0,0 +1 @@
+*.nl          text eol=lf

.github/CODEOWNERS

@@ -0,0 +1,48 @@
+# Code owners will be used to identify maintainers, notably
+# for developmental and contributed code
+
+# Overall (non-specific)
+* @lbianchi-lbl @ksbeattie
+# packaging
+setup.py @lbianchi-lbl @ksbeattie
+requirements*.txt @lbianchi-lbl @ksbeattie 
+
+# CI infra
+.github/ @lbianchi-lbl @ksbeattie 
+
+# testing
+pytest*.ini @lbianchi-lbl
+conftest.py @lbianchi-lbl
+# Docs - need to identify owner(s)
+
+# Commands
+/idaes/commands/ @eslickj @dangunter
+
+# Core
+/idaes/core/ @andrewlee94 @eslickj @lbianchi-lbl
+/idaes/core/dmf/ @dangunter
+/idaes/core/surrogate/ @andrewlee94 @carldlaird @bpaul4
+/idaes/core/ui/ @dangunter
+
+# Models
+/idaes/models/ @andrewlee94 @eslickj
+
+# Apps - each package needs a maintainer
+/idaes/apps/caprese/ @Robbybp
+/idaes/apps/grid_integration/ @jsiirola @adowling2
+#     /idaes/apps/matopt/ TBD
+/idaes/apps/nmpc/ @Robbybp
+/idaes/apps/uncertainty_propagation/ @adowling2
+
+# Models Extra - each package needs a maintainer
+#     /idaes/models_extra/carbon_capture TBD
+/idaes/models_extra/column_models/ @aostace01 @agarciadiego
+/idaes/models_extra/gas_distribution/ @Robbybp
+/idaes/models_extra/gas_solids_contactors/ @aostace01
+/idaes/models_extra/power_generation/ @eslickj @MAZamarripa
+/idaes/models_extra/power_generation/costing/ @MAZamarripa @AlexNoring
+/idaes/models_extra/power_generation/properties/ @eslickj @AlexNoring
+/idaes/models_extra/power_generation/unit_models/soc_submodels/ @eslickj @dallan-keylogic
+
+# Scripts
+/scripts/colab_helper.py @adowling2

.github/actions/run-examples/action.yml

@@ -1,56 +1,78 @@
 name: Run IDAES examples
 description: Run IDAES examples
+
 inputs:
-  git-ref:
-    description: git ref to fetch for the examples
+
+  reinstall-target:
+    description: Installation variant or pip install arguments to use to install IDAES
+    required: true
+
+  examples-repository:
+    description: Full name (owner/repo) of the repository to use for the examples
+    required: false
+    default: IDAES/examples
+
+  examples-ref:
+    description: git ref to use for the examples
     required: false
     default: main
+
+  notebook-cell-exec-timeout:
+    description: Maximum time in s allowed for each individual notebook cell to execute before an error is raised
+    required: true
+
   working-dir:
     description: Working directory where the examples will be run
     required: false
     default: ${{ runner.temp }}
-  config-file-name:
-    description: >-
-      Name of the build script config file which will be copied to the working dir.
-      The source directory is assumed to be this action's directory.
+
+  pytest-extra-flags:
+    description: Extra flag to pass to pytest
     required: false
-    default: build-config-for-idaes-ci.yml 
+    default: ''
 
 runs:
   using: composite
   steps:
-    - name: Fetch examples
-      working-directory: ${{ inputs.working-dir }}
-      shell: bash
+
+    - name: Install examples repository as Python package
+      # -l needed to be able to run this step correctly within a Conda environment
+      shell: bash -l {0}
+      env:
+        PIP_PROGRESS_BAR: 'off'
+        _pip_extras: '[jb,dev]'
+        _pip_install_url: ${{ github.server_url}}/${{ inputs.examples-repository }}@${{ inputs.examples-ref }}
       run: |
-        rm -rf *
-        git clone https://github.com/IDAES/examples-pse.git --branch ${{ inputs.git-ref }} .
-        git log -n 1
-    - name: Install examples package and/or dependencies
-      working-directory: ${{ inputs.working-dir }}
+        echo '::group::Output of `pip install idaes_examples`'
+        pip install "idaes_examples${_pip_extras} @ git+${_pip_install_url}"
+        echo '::endgroup::'
+        # uninstall IDAES if installed through idaes_examples so that the version under test can be used
+        pip uninstall --yes idaes-pse
+
+    - name: Reinstall IDAES
       shell: bash -l {0}
       run: |
-        pip install --progress-bar off --editable .
-    - name: Set up config file
-      shell: bash
-      # issue the cp command from the (source) dir where the config file is stored
-      # to avoid issues with path separators on Windows
+        echo '::group::Output of `pip install ${{ inputs.reinstall-target }}`'
+        pip install ${{ inputs.reinstall-target }}
+        echo '::endgroup::'
+
+    - name: Run notebooks through pytest
       working-directory: ${{ inputs.working-dir }}
+      shell: bash -l {0}
       env:
-        config_file_name: ${{ inputs.config-file-name }}
+        PYTHONPATH: ${{ github.action_path }}
+        PYTEST_ADDOPTS: >-
+          --color=yes
+          --durations=0
+          -p no:xdist
+          --nbmake-timeout=${{ inputs.notebook-cell-exec-timeout }}
+          ${{ inputs.pytest-extra-flags }}
       run: |
-        config_file_dst_dir="$PWD"
-        config_file_src_dir="$GITHUB_ACTION_PATH"
-        cd "$config_file_src_dir"
-        ls -la "$PWD"
-        cp "$config_file_name" "$config_file_dst_dir"
-        echo "config_file_name=$config_file_name" >> "$GITHUB_ENV"
-    - name: Run build.py script
-      working-directory: ${{ inputs.working-dir }}
-      # -l needed to be able to run this step correctly within a Conda environment
+        rm -rf *  # ensure directory is empty
+        python -m examples_for_idaes_ci
+
+    - name: Display content of working directory after run
       shell: bash -l {0}
+      working-directory: ${{ inputs.working-dir }}
       run: |
-        echo "::group::Contents of config file $config_file_name"
-        cat "$config_file_name"
-        echo "::endgroup::"
-        python build.py -v --config "$config_file_name" --test
+        ls -Ralh

.github/actions/run-examples/examples_for_idaes_ci.py

@@ -0,0 +1,140 @@
+"""
+pytest plugin for testing IDAES "through" IDAES/examples within the IDAES/idaes-pse CI
+"""
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+import fnmatch
+import logging
+import os
+from pathlib import Path
+import sys
+from typing import List
+
+import pytest
+
+import idaes_examples
+from idaes_examples import notebooks
+from idaes_examples import build
+
+
+matchmarker = pytest.StashKey()
+marked = pytest.StashKey()
+
+
+def _matches_pattern(item: pytest.Item, pattern: str) -> bool:
+    to_match = os.fspath(item.path)
+    return fnmatch.fnmatch(to_match, pattern)
+
+
+def pytest_configure(config: pytest.Config):
+    config.stash[matchmarker] = {
+        "*/held/*": pytest.mark.xfail(run=False, reason="notebook has 'held' status"),
+        "*/archive/*": pytest.mark.skip(reason="notebook is archived"),
+        # TODO: Need to fix this once the Python 3.11 issue is resolved in tensorflow
+        "*/surrogates/best_practices_optimization*": pytest.mark.xfail(
+            condition=sys.version_info > (3, 11),
+            run=True,
+            strict=False,
+            reason="tensorflow ImportError on 3.11",
+        ),
+        "*/surrogates/omlt/keras_flowsheet_optimization*": pytest.mark.xfail(
+            condition=sys.version_info > (3, 11),
+            run=True,
+            strict=False,
+            reason="tensorflow ImportError on 3.11",
+        ),
+    }
+    config.stash[marked] = []
+
+
+def pytest_sessionstart(session: pytest.Session):
+    build.preprocess(session.config.rootpath)
+
+
+def pytest_ignore_collect(collection_path: Path, config: pytest.Config):
+    if "_dev" in collection_path.parts:
+        return True
+    if not collection_path.is_file():
+        return
+    if collection_path.suffix == ".py":
+        # specifically ignore python files
+        return True
+    if not collection_path.match("**/*_test.ipynb"):
+        return True
+
+
+def pytest_collection_modifyitems(config: pytest.Config, items):
+    for item in items:
+        for pattern, marker in config.stash[matchmarker].items():
+            if _matches_pattern(item, pattern):
+                item.add_marker(marker)
+                config.stash[marked].append((item, pattern, marker))
+
+
+def pytest_report_collectionfinish(config: pytest.Session):
+    lines = []
+    for item, pattern, marker in config.stash[marked]:
+        lines += [f"\t{item.nodeid=}\t{pattern=}\t{marker.mark}"]
+    if lines:
+        lines.insert(0, "The following items have been marked:")
+    return lines
+
+
+@contextmanager
+def _temp_cwd(path: Path):
+    orig_wdir = Path.cwd()
+    try:
+        os.chdir(path)
+        yield path
+    finally:
+        os.chdir(orig_wdir)
+
+
+def run_pytest(
+    rootdir: Path,
+    args: List[str],
+    ignore_conftest: bool = True,
+    ignore_inifile: bool = True,
+    **kwargs,
+):
+    args = [
+        str(rootdir),
+        f"--rootdir={rootdir}",
+        *args,
+    ]
+
+    if ignore_conftest:
+        args += ["--noconftest"]
+    if ignore_inifile:
+        empty_file_for_ignoring = rootdir / "__empty__"
+        empty_file_for_ignoring.write_text("")
+        args += ["-c", empty_file_for_ignoring]
+
+    with _temp_cwd(rootdir) as p:
+        res = pytest.main(
+            args,
+            **kwargs,
+        )
+
+    return res
+
+
+def main(args):
+    rootdir = Path(idaes_examples.__path__[0])
+
+    res = run_pytest(
+        rootdir,
+        [
+            "--nbmake",
+            *args,
+        ],
+        ignore_conftest=True,
+        ignore_inifile=True,
+        plugins=[__name__],
+    )
+
+    return res
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))