Update actions in workflow and improve parallelisation and cache hand…

…ling (#298) Changelog === In general, the versions of the actions were updated for checkout, cache, download and upload artifacts. Aditionally, the parallel build with a static value in an environment variable `-j2` was changed to the `--parallel $(nproc)` argument to the command to pick automatically the value. In particular, other changes were done: - `build_test_linux_fedora_minimal.yaml` - the build cache is updated with a new name to preserve for the same reference (branch, push) and not for the same commit - `build_test_linux_fedora.yaml` - setup a build cache for dpsimpy & examples - archive in a different cache (not overwriting the others) per commit sha - changes to the archiving of build of cxx examples - input from the previous stage and recompile and save for the current compilation - jupyter notebook tests were parallelized - the package pytest-xdist is installed - cache is changed to separate per commit sha - comparison of results - previous cached result is used - test of binaries - the output of the cxx compilation is used as cache - villas examples - the needs is changed to examples (dpsimpy probably worked due old overwrites in the cache, when chaining the compilation outputs it started failing) - `build_test_linux_rocky.yaml` - added cache to the clang compilation - added cache to the profiling compilation - `build_test_windows.yaml` - added cache to the build directory per reference - `documentation-fein.yaml` - changed node to v20 - updated hugo and gh-pages actions - `run_and_profile_example.yaml` - changed the container to sogno/dpsim.dev-rocky (we were still on centos) - `run_villas_example.yaml` - changed the cache to the cxx examples build
sogno-platform · Jun 10, 2024 · a95d999 · a95d999
2 parents 37442dc + 914a422
commit a95d999
Show file tree

Hide file tree

Showing 10 changed files with 134 additions and 123 deletions.
diff --git a/.github/workflows/build_test_linux_fedora.yaml b/.github/workflows/build_test_linux_fedora.yaml
@@ -7,18 +7,24 @@ on:
 
 jobs:
   linux-fedora-dpsimpy:
-    name: Build dpsimpy on Fedora Linux
+    name: Build dpsimpy and tests examples on Fedora Linux
     runs-on: ubuntu-latest
     container: sogno/dpsim:dev
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
     - name: Create Build Environment
       run: mkdir build
 
+    - name: Setup build directory cache
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/build
+        key: build-fedora-dir-cache-${{ github.ref }}
+
     - name: Configure CMake
       shell: bash
       working-directory: ${{ github.workspace }}/build
@@ -27,15 +33,13 @@ jobs:
     - name: Build dpsimpy and test examples
       shell: bash
       working-directory: ${{ github.workspace }}/build
-      run: cmake --build . --target dpsimpy --target tests --target dpsimpyvillas
-      env:
-        MAKEFLAGS: "-j2"
+      run: cmake --build . --target dpsimpy --target tests --target dpsimpyvillas --parallel $(nproc)
 
     - name: Archive build directory
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         path: ${{ github.workspace }}/build
-        name: build-cache
+        name: build-fedora-examples-cache-${{ github.sha }}
         retention-days: 1
 
   linux-fedora-examples:
@@ -45,28 +49,26 @@ jobs:
     container: sogno/dpsim:dev
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
     - name: Restore build archive
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
-       name: build-cache
+       name: build-fedora-examples-cache-${{ github.sha }}
        path: ${{ github.workspace }}/build
 
     - name: Build every target
       shell: bash
       working-directory: ${{ github.workspace }}/build
-      run: cmake --build .
-      env:
-        MAKEFLAGS: "-j2"
+      run: cmake --build . --parallel $(nproc)
 
     - name: Archive build directory
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         path: ${{ github.workspace }}/build
-        name: build-cache
+        name: build-cache-examples-cpp-${{ github.sha }}
         retention-days: 1
 
 ## Tests ##
@@ -77,12 +79,12 @@ jobs:
     container: sogno/dpsim:dev
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Restore build archive
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
-       name: build-cache
+       name: build-fedora-examples-cache-${{ github.sha }}
        path: ${{ github.workspace }}/build
 
     - name: Prepare binary permissions
@@ -99,42 +101,46 @@ jobs:
       working-directory: ${{ github.workspace }}
       run: ls -l ./build/dpsim/examples/cxx/
 
+    - name: Install parallel testing
+      shell: bash
+      run: pip install pytest-xdist
+
     - name: Run pytest
       shell: bash
       working-directory: ${{ github.workspace }}
       env:
         PYTHONPATH: "${{ github.workspace }}/build"
       run: |
         cp -r python/src/dpsim build/
-        pytest -v examples/Notebooks
+        pytest -v examples/Notebooks -n auto
 
     - name: Archive notebook outputs
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: pytest-output
-        path: outputs/examples/Notebooks/
+        name: pytest-output-${{ github.sha }}
+        path: outputs/
 
   compare-notebook-results:
     name: Compare Notebook results
     runs-on: ubuntu-latest
     needs: [test-jupyter-notebooks]
     steps:
     - name: Download new notebook results
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
-       name: pytest-output
+       name: pytest-output-${{ github.sha }}
        path: ${{ github.workspace }}/notebooks-new
 
     - name: Download master notebook results
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ github.workspace }}/notebooks-master
         key: notebook-output-cache-master-${{ github.sha }}
         restore-keys: |
           notebook-output-cache-master-
 
     - name: Download previous commit notebook results
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ github.workspace }}/notebooks-previous
         key: notebook-output-cache-commit-${{ github.ref }}-${{ github.sha }}
@@ -165,15 +171,15 @@ jobs:
         cp -R ${{ github.workspace }}/notebooks-new ${{ github.workspace }}/notebooks-master
 
   test-binaries:
-    name: Execute Example
+    name: Execute WSCC 9 bus example
     runs-on: ubuntu-latest
     container: sogno/dpsim:dev
     needs: [linux-fedora-examples]
     steps:
     - name: Restore build archive
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
-       name: build-cache
+       name: build-cache-examples-cpp-${{ github.sha }}
        path: ${{ github.workspace }}/build
 
     - name: Prepare binary permissions
@@ -194,15 +200,15 @@ jobs:
     #needs: [linux-fedora]
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Run cppcheck
       working-directory: ${{ github.workspace }}
       shell: bash
       id: cppcheck
       run: |
         set -o pipefail
-        cppcheck --max-configs=32 -j 32 --inline-suppr --error-exitcode=1 -q --enable=warning,performance,portability,information,missingInclude --std=c++17 -I dpsim/include/ -I dpsim-models/include/ -I dpsim-villas/include dpsim/src/ dpsim-models/src/ dpsim-villas/src/ 2>&1 | tee cppcheck-output.log
+        cppcheck --max-configs=32 -j $(nproc) --inline-suppr --error-exitcode=1 -q --enable=warning,performance,portability,information,missingInclude --std=c++17 -I dpsim/include/ -I dpsim-models/include/ -I dpsim-villas/include dpsim/src/ dpsim-models/src/ dpsim-villas/src/ 2>&1 | tee cppcheck-output.log
       continue-on-error: true
 
     - name: Print cppcheck errors
@@ -262,7 +268,7 @@ jobs:
 
   test-villas-examples-1:
     name: Run dpsim-mqtt VILLASnode example
-    needs: [linux-fedora-dpsimpy]
+    needs: [linux-fedora-examples]
     uses: ./.github/workflows/run_villas_example.yaml
     with:
       compose_file_path: examples/villas/docker-compose-tests
@@ -274,7 +280,7 @@ jobs:
 
   test-villas-examples-2:
     name: Run dpsim-mqtt-import-export VILLASnode example
-    needs: [linux-fedora-dpsimpy]
+    needs: [linux-fedora-examples]
     uses: ./.github/workflows/run_villas_example.yaml
     with:
       compose_file_path: examples/villas/docker-compose-tests
@@ -284,7 +290,7 @@ jobs:
 
   test-villas-examples-3:
     name: Run dpsim-mqtt-import-export-MIMO VILLASnode example
-    needs: [linux-fedora-dpsimpy]
+    needs: [linux-fedora-examples]
     uses: ./.github/workflows/run_villas_example.yaml
     with:
       compose_file_path: examples/villas/docker-compose-tests
@@ -295,7 +301,7 @@ jobs:
 
   test-villas-examples-4:
     name: Run dpsim-file VILLASnode example
-    needs: [linux-fedora-dpsimpy]
+    needs: [linux-fedora-examples]
     uses: ./.github/workflows/run_villas_example.yaml
     with:
       compose_file_path: examples/villas/docker-compose-tests
@@ -306,7 +312,7 @@ jobs:
 
   test-villas-examples-5:
     name: Run mqtt-cigre-mv-pf-profiles VILLASnode example
-    needs: [linux-fedora-dpsimpy]
+    needs: [linux-fedora-examples]
     uses: ./.github/workflows/run_villas_example.yaml
     with:
       compose_file_path: examples/villas/docker-compose-tests

diff --git a/.github/workflows/build_test_linux_fedora_minimal.yaml b/.github/workflows/build_test_linux_fedora_minimal.yaml
@@ -12,13 +12,19 @@ jobs:
     container: sogno/dpsim:dev-minimal
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
     - name: Create Build Environment
       run: mkdir build
 
+    - name: Cache build directory
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/build
+        key: build-cache-fedora-minimal-${{ github.ref }}
+
     - name: Configure CMake
       shell: bash
       working-directory: ${{ github.workspace }}/build
@@ -27,14 +33,7 @@ jobs:
     - name: Build every target
       shell: bash
       working-directory: ${{ github.workspace }}/build
-      run: cmake --build .
-      env:
-        MAKEFLAGS: "-j2"
+      run: cmake --build . --parallel $(nproc)
 
-    - name: Cache build directory
-      uses: actions/cache@v3
-      with:
-        path: ${{ github.workspace }}/build
-        key: build-cache-fedora-minimal-${{ github.sha }}
 ## Tests ##
   #TODO
diff --git a/.github/workflows/build_test_linux_rocky.yaml b/.github/workflows/build_test_linux_rocky.yaml
@@ -12,13 +12,19 @@ jobs:
     container: sogno/dpsim:dev-rocky
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
     - name: Create Build Environment
       run: mkdir build
 
+    - name: Cache build directory
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/build
+        key: build-cache-rocky-${{ github.ref }}
+
     - name: Configure CMake
       shell: bash
       working-directory: ${{ github.workspace }}/build
@@ -27,29 +33,27 @@ jobs:
     - name: Build every target
       shell: bash
       working-directory: ${{ github.workspace }}/build
-      run: cmake --build .
-      env:
-        MAKEFLAGS: "-j2"
-
-    - name: Cache build directory
-      uses: actions/cache@v3
-      with:
-        path: ${{ github.workspace }}/build
-        key: build-cache-rocky-${{ github.sha }}
+      run: cmake --build . --parallel $(nproc)
 
   linux-rocky-clang:
     name: Build on Rocky Linux using Clang
     runs-on: ubuntu-latest
     container: sogno/dpsim:dev-rocky
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
     - name: Create Build Environment
       run: mkdir build
 
+    - name: Cache build directory
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/build
+        key: build-cache-rocky-clang-${{ github.ref }}
+
     - name: Configure CMake
       shell: bash
       working-directory: ${{ github.workspace }}/build
@@ -58,29 +62,28 @@ jobs:
     - name: Build every target
       shell: bash
       working-directory: ${{ github.workspace }}/build
-      run: cmake --build .
-      env:
-        MAKEFLAGS: "-j2"
+      run: cmake --build . --parallel $(nproc)
 
-    - name: Cache build directory
-      uses: actions/cache@v3
-      with:
-        path: ${{ github.workspace }}/build
-        key: build-cache-rocky-${{ github.sha }}
 
   profiling:
     name: Build with Profiling options
     runs-on: ubuntu-latest
     container: sogno/dpsim:dev-rocky
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
     - name: Create Build Environment
       run: mkdir build
 
+    - name: Cache build directory
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/build
+        key: build-cache-rocky-profiling-${{ github.ref }}
+
     - name: Configure CMake
       shell: bash
       working-directory: ${{ github.workspace }}/build
@@ -89,15 +92,8 @@ jobs:
     - name: Build every target
       shell: bash
       working-directory: ${{ github.workspace }}/build
-      run: cmake --build .
-      env:
-        MAKEFLAGS: "-j2"
+      run: cmake --build . --parallel $(nproc)
 
-    - name: Cache build directory
-      uses: actions/cache@v3
-      with:
-        path: ${{ github.workspace }}/build
-        key: build-cache-rocky-profiling-${{ github.sha }}
 
 ## Tests ##
   test-examples-1: