Merge remote-tracking branch 'origin/candidate-9.8.x'

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>
ghalliday · Nov 26, 2024 · 3f7492d · 3f7492d
2 parents e92a2bc + ab280fe
commit 3f7492d
Show file tree

Hide file tree

Showing 13 changed files with 297 additions and 146 deletions.
diff --git a/.github/actions/deploy-hpcc-k8s/action.yaml b/.github/actions/deploy-hpcc-k8s/action.yaml
@@ -23,13 +23,13 @@ runs:
       shell: bash
 
     - name: Download Package
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: ${{ inputs.asset-name }}
         path: ${{ inputs.asset-name }}
 
     - name: Download Support Files
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: ${{ inputs.asset-name }}-support-files
         path: ${{ inputs.asset-name }}-support-files

diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml
@@ -40,11 +40,15 @@ jobs:
       candidate_branch: ${{ steps.vars.outputs.candidate_branch }}
       cmake_docker_config: ${{ steps.vars.outputs.cmake_docker_config }}
       gpg_import: ${{ steps.vars.outputs.gpg_import }}
+      folder_prettygitlogs: ${{ steps.vars.outputs.folder_prettygitlogs }}
+      folder_eclide: ${{ steps.vars.outputs.folder_eclide }}
     steps:
       - name: Calculate vars
         id: vars
         run: |
           echo "folder_platform=${{ github.workspace }}/HPCC-Platform" >> $GITHUB_OUTPUT
+          echo "folder_eclide"=${{ github.workspace }}/ECLIDE >> $GITHUB_OUTPUT
+          echo "folder_prettygitlogs"=${{ github.workspace }}/PrettyGitLogs >> $GITHUB_OUTPUT
           echo 'mount_platform=source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached' >> $GITHUB_OUTPUT
           echo "folder_ln=${{ github.workspace }}/LN" >> $GITHUB_OUTPUT
           echo 'mount_ln=source="${{ github.workspace }}/LN",target=/hpcc-dev/LN,type=bind,consistency=cached' >> $GITHUB_OUTPUT
@@ -695,3 +699,111 @@ jobs:
         with:
           name: windows-2022-logs
           path: ./ECLIDE/build/**/*.log
+
+
+  changelogs:
+    name: Generate Changelogs
+    needs: preamble
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout HPCC-Platform
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.preamble.outputs.community_ref }}
+          submodules: recursive
+          path: ${{ needs.preamble.outputs.folder_platform }}
+          fetch-tags: true
+          fetch-depth: 0
+      - name: Checkout LN
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/LN
+          token: ${{ secrets.LNB_TOKEN }}
+          ref: ${{ needs.preamble.outputs.internal_ref }}
+          submodules: recursive
+          path: ${{ needs.preamble.outputs.folder_ln }}
+          fetch-tags: true
+          fetch-depth: 0
+      - name: Checkout ECLIDE
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/ECLIDE
+          ref: ${{ needs.preamble.outputs.eclide_ref }}
+          submodules: recursive
+          path: ${{ needs.preamble.outputs.folder_eclide }}
+          fetch-tags: true
+          fetch-depth: 0
+      - name: Checkout PrettyGitLogs
+        uses: actions/checkout@v4
+        with:
+          repository: hpcc-systems/PrettyGitLogs
+          ref: master
+          path: ${{ needs.preamble.outputs.folder_prettygitlogs }}
+      - name: Get Previous Versions
+        id: get_version
+        shell: bash
+        working-directory: ${{ needs.preamble.outputs.folder_platform }}
+        run: |
+          tag=$(git describe --exact-match --tags 2>/dev/null)
+          version=$(echo $tag | cut -d_ -f2)
+          major=$(echo $version | cut -d. -f1)
+          minor=$(echo $version | cut -d. -f2)
+          point=$(echo $version | cut -d. -f3 | cut -d- -f1)
+          type=$(echo $version | cut -d- -f2)
+          if [[ $point -eq 0 ]]; then
+            if [[ $minor -eq 0 ]]; then
+              major=$((major - 1))
+              previous_rc=$(git tag --create-reflog --sort=-v:refname | grep -E "^community_$major\.[0-9]*\.[0-9]*-rc" | head -n 1 | cut -d_ -f2 )
+              previous_gold=$(git tag --create-reflog --sort=-v:refname | grep -E "^community_$major\.[0-9]*\.[0-9]*-[0-9]" | head -n 1 | cut -d_ -f2 )
+            else
+              minor=$((minor - 2))
+              previous_rc=$(git tag --create-reflog --sort=-v:refname | grep -E "^community_$major\.$minor\.[0-9]*-rc" | head -n 1 | cut -d_ -f2 )
+              previous_gold=$(git tag --create-reflog --sort=-v:refname | grep -E "^community_$major\.$minor\.[0-9]*-[0-9]" | head -n 1 | cut -d_ -f2 )
+            fi
+          else
+            point=$((point - 2))
+            previous_rc=$(git tag --create-reflog --sort=-v:refname | grep -E "^community_$major\.$minor\.$point-rc" | head -n 1 | cut -d_ -f2 )
+            previous_gold=$(git tag --create-reflog --sort=-v:refname | grep -E "^community_$major\.$minor\.$point-[0-9]" | head -n 1 | cut -d_ -f2)
+          fi
+          echo "current_version=$version" >> $GITHUB_OUTPUT
+          echo "previous_rc_version=$previous_rc" >> $GITHUB_OUTPUT
+          echo "previous_gold_version=$previous_gold" >> $GITHUB_OUTPUT
+          echo "type=$type" >> $GITHUB_OUTPUT
+      - name: Print vars
+        run: |
+          echo "${{ toJSON(steps.get_version.outputs) }}"
+      - name: Generate Changelogs
+        shell: bash
+        working-directory: ${{ needs.preamble.outputs.folder_prettygitlogs }}
+        run: |
+          # run setup for perl environment
+          ./prettyLogs_setup.sh
+
+          # provide credentials
+          sed -i -e '/^JIRAUser/c\JIRAUser=${{ secrets.JIRA_USERNAME}}' prettylogs.conf
+          sed -i -e '/^JIRAPW/c\JIRAPW=${{ secrets.JIRA_TOKEN }}' prettylogs.conf
+ 
+          # Generate Changelogs
+          if [[ ! "${{ steps.get_version.outputs.type }}" == *"rc"* ]]; then
+            perl ./prettyLogs.pl -bt community_${{ steps.get_version.outputs.current_version }} -et community_${{ steps.get_version.outputs.previous_gold_version }} -repo ${{ needs.preamble.outputs.folder_platform }} -sort -html community_${{ steps.get_version.outputs.current_version }}
+            perl ./prettyLogs.pl -bt internal_${{ steps.get_version.outputs.current_version }} -et internal_${{ steps.get_version.outputs.previous_gold_version }} -repo ${{ needs.preamble.outputs.folder_ln }} -sort -html internal_${{ steps.get_version.outputs.current_version }}
+            perl ./prettyLogs.pl -bt eclide_${{ steps.get_version.outputs.current_version }} -et eclide_${{ steps.get_version.outputs.previous_gold_version }} -repo ${{ needs.preamble.outputs.folder_eclide }} -sort -html eclide_${{ steps.get_version.outputs.current_version }}
+          elif [ "${{ steps.get_version.outputs.type }}" == "rc1" ]; then
+            perl ./prettyLogs.pl -bt community_${{ steps.get_version.outputs.current_version }} -et community_${{ steps.get_version.outputs.previous_gold_version }} -repo ${{ needs.preamble.outputs.folder_platform }} -sort -html community_${{ steps.get_version.outputs.current_version }}
+            perl ./prettyLogs.pl -bt internal_${{ steps.get_version.outputs.current_version }} -et internal_${{ steps.get_version.outputs.previous_gold_version }} -repo ${{ needs.preamble.outputs.folder_ln }} -sort -html internal_${{ steps.get_version.outputs.current_version }}
+            perl ./prettyLogs.pl -bt eclide_${{ steps.get_version.outputs.current_version }} -et eclide_${{ steps.get_version.outputs.previous_gold_version }} -repo ${{ needs.preamble.outputs.folder_eclide }} -sort -html eclide_${{ steps.get_version.outputs.current_version }}
+          else
+            perl ./prettyLogs.pl -bt community_${{ steps.get_version.outputs.current_version }} -et community_${{ steps.get_version.outputs.previous_rc_version }} -repo ${{ needs.preamble.outputs.folder_platform }} -sort -html community_${{ steps.get_version.outputs.current_version }}
+            perl ./prettyLogs.pl -bt internal_${{ steps.get_version.outputs.current_version }} -et internal_${{ steps.get_version.outputs.previous_rc_version }} -repo ${{ needs.preamble.outputs.folder_ln }} -sort -html internal_${{ steps.get_version.outputs.current_version }}
+            perl ./prettyLogs.pl -bt eclide_${{ steps.get_version.outputs.current_version }} -et eclide_${{ steps.get_version.outputs.previous_rc_version }} -repo ${{ needs.preamble.outputs.folder_eclide }} -sort -html eclide_${{ steps.get_version.outputs.current_version }}
+          fi
+      - name: Upload Changelogs
+        uses: ncipollo/release-action@v1.14.0
+        with:
+          allowUpdates: true
+          generateReleaseNotes: false
+          prerelease: ${{ contains(github.ref, '-rc') }}
+          artifacts: |
+            ${{ needs.preamble.outputs.folder_prettygitlogs }}/community_${{ steps.get_version.outputs.current_version }}.html
+            ${{ needs.preamble.outputs.folder_prettygitlogs }}/internal_${{ steps.get_version.outputs.current_version }}.html
+            ${{ needs.preamble.outputs.folder_prettygitlogs }}/eclide_${{ steps.get_version.outputs.current_version }}.html
diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
@@ -158,7 +158,7 @@ jobs:
 
       - name: Upload Package
         if: ${{ inputs.upload-package == true }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.asset-name }}
           path: |
@@ -168,7 +168,7 @@ jobs:
 
       - name: Upload Support Files
         if: ${{ inputs.upload-package == true }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.asset-name }}-support-files
           path: |
@@ -178,7 +178,7 @@ jobs:
 
       - name: Upload ECL Watch UI Test Files
         if: ${{ inputs.upload-package == true }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.asset-name }}-ecl_watch_ui_tests
           path: |
@@ -187,7 +187,7 @@ jobs:
 
       - name: Upload Error Logs
         if: ${{ failure() || cancelled() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.os }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }}-logs
           path: ${{ github.workspace }}/build/**/*.log
diff --git a/.github/workflows/build-gh_runner.yml b/.github/workflows/build-gh_runner.yml
@@ -214,7 +214,7 @@ jobs:
 
       - name: Upload Package
         if: ${{ inputs.upload-package == true }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.asset-name }}
           path: |
@@ -224,7 +224,7 @@ jobs:
 
       - name: Upload Support Files
         if: ${{ inputs.upload-package == true }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.asset-name }}-support-files
           path: |
@@ -234,7 +234,7 @@ jobs:
 
       - name: Upload UI Test Files
         if: ${{ inputs.upload-package == true }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.asset-name }}-ui_test-files
           path: |
@@ -243,7 +243,7 @@ jobs:
 
       - name: Upload Error Logs
         if: ${{ failure() || cancelled() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.os }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }}-logs
           path: ${{ github.workspace }}/build/**/*.log
diff --git a/.github/workflows/test-smoke-gh_runner.yml b/.github/workflows/test-smoke-gh_runner.yml
@@ -53,13 +53,13 @@ jobs:
           sudo rm -rf /usr/local/lib/android
 
       - name: Download Package
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ inputs.asset-name }}
           path: ${{ inputs.asset-name }}
 
       - name: Download Support Files
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ inputs.asset-name }}-support-files
           path: ${{ inputs.asset-name }}-support-files
@@ -132,7 +132,7 @@ jobs:
 
       - name: regression-run-logs-artifact
         if: ${{ failure() || cancelled() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: test-smoke-${{ inputs.asset-name }}-${{ matrix.engine }}-${{ steps.vars.outputs.matrix-setname }}
           path: |

diff --git a/.github/workflows/test-ui-gh_runner.yml b/.github/workflows/test-ui-gh_runner.yml
@@ -34,7 +34,7 @@ jobs:
           sudo rm -rf /usr/local/lib/android
 
       - name: Download UI Test Files
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ inputs.asset-name }}-ecl_watch_ui_tests
           path: ${{ inputs.asset-name }}-ecl_watch_ui_tests
@@ -74,7 +74,7 @@ jobs:
 
       - name: Download Package
         if: steps.check.outputs.runtests
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ inputs.asset-name }}
           path: ${{ inputs.asset-name }}
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload ECL Watch UI Test Logs To Artifact
         if: ${{ failure() || cancelled() || env.uploadArtifact == 'true' }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.asset-name }}-ecl_watch_ui_tests
           path: |

diff --git a/.github/workflows/test-unit-gh_runner.yml b/.github/workflows/test-unit-gh_runner.yml
@@ -30,7 +30,7 @@ jobs:
           sudo rm -rf /usr/local/lib/android
 
       - name: Download Package
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ inputs.asset-name }}
           path: ${{ inputs.asset-name }}
@@ -58,7 +58,7 @@ jobs:
 
       - name: unittests-logs-artifact
         if: ${{ failure() || cancelled() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: test-unit-${{ inputs.asset-name }}
           path: |

diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp
@@ -14585,6 +14585,11 @@ void executeThorGraph(const char * graphName, IConstWorkUnit &workunit, const IP
     // NB: check for expected success state (WUStateWait). If any other state, abort.
     {
         Owned<IWorkUnit> w = &workunit.lock();
+        //If the thor instance crashed, make sure that the workunit is no longer associated with it - otherwise a
+        //failure clause that causes a graph to run can abort because the instances has stopped.
+        if (w->getEngineSession() > 0)
+            w->setEngineSession(-1);
+
         WUState state = w->getState();
         if (WUStateWait != state) // expected state from successful Thor run from above
         {