From d8a68ec1c48149e2161dbfe1c93407bae43a1314 Mon Sep 17 00:00:00 2001 From: aboyko Date: Wed, 10 Jul 2024 13:40:51 -0400 Subject: [PATCH] [GHA] Eclipse LS extensions and distro snapshot to R2 bucket [GHA] Revert to main branch --- .../workflows/eclipse-ls-extensions-build.yml | 23 ++--- .github/workflows/gen-nighly-downloads.yml | 30 +++--- .../gh-hosted-eclipse-distro-build.yml | 97 +++++-------------- .../snapshot-eclipse-ls-extensions-build.yml | 1 + 4 files changed, 47 insertions(+), 104 deletions(-) diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index c3ca21adf3..61899ddff0 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -25,6 +25,13 @@ on: description: version of sts4 ls extensions required: false type: string +env: + AWS_ACCESS_KEY_ID: ${{ secrets.CDN_S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CDN_S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT_URL_S3: ${{ secrets.CDN_S3_ENDPOINT }} + AWS_S3_BUCKET: ${{ secrets.CDN_BUCKET }} + DOWNLOAD_URL_ROOT: https://cdn-test.spring.io/spring-tools jobs: @@ -74,24 +81,14 @@ jobs: else ./mvnw --batch-mode clean install -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Dtycho.localArtifacts=ignore fi - - name: Cloudgate S3 Configuration - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a - with: - aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} - role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} - role-session-name: ${{ github.run_id }} - aws-region: us-east-1 - role-duration-seconds: 900 - role-skip-session-tagging: true - name: Upload P2 Repo to S3 and Akamai id: upload-p2-to-akamai run: | p2_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt` echo "p2_path=$p2_path" >> $GITHUB_OUTPUT echo "P2 path: ${p2_path}" - aws s3 rm s3://tools-spring-io/${p2_path}/ --recursive - aws s3 cp ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/ s3://tools-spring-io/${p2_path}/ --recursive --no-progress + aws s3 rm s3://$AWS_S3_BUCKET/${p2_path}/ --recursive + aws s3 cp ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/ s3://$AWS_S3_BUCKET/${p2_path}/ --recursive --no-progress # - name: Verify Uploaded P2 Repo # id: verify-p2-on-akamai # run: | @@ -106,7 +103,7 @@ jobs: if: ${{ inputs.build_type != 'snapshot' }} id: compute-invalid-urls run: | - invalid_urls=`aws s3 ls s3://tools-spring-io/${{ steps.upload-p2-to-akamai.outputs.p2_path }}/ --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -` + invalid_urls=`aws s3 ls s3://$AWS_S3_BUCKET/${{ steps.upload-p2-to-akamai.outputs.p2_path }}/ --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -` echo "invalid_urls=$invalid_urls" >> $GITHUB_OUTPUT - name: Verify Eclipse LS Extensions on e433 if: ${{ inputs.build_type == 'snapshot' }} diff --git a/.github/workflows/gen-nighly-downloads.yml b/.github/workflows/gen-nighly-downloads.yml index 9ee186075f..1b6108774d 100644 --- a/.github/workflows/gen-nighly-downloads.yml +++ b/.github/workflows/gen-nighly-downloads.yml @@ -3,47 +3,45 @@ name: Generate Nightly Downloads HTML # configure manual trigger on: workflow_dispatch: +env: + AWS_ACCESS_KEY_ID: ${{ secrets.CDN_S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CDN_S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT_URL_S3: ${{ secrets.CDN_S3_ENDPOINT }} + AWS_S3_BUCKET: ${{ secrets.CDN_BUCKET }} + DOWNLOAD_URL_ROOT: https://cdn-test.spring.io/spring-tools jobs: gen_html: runs-on: ubuntu-latest steps: - - name: Cloudgate S3 Configuration - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a - with: - aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} - role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} - role-session-name: ${{ github.run_id }} - aws-region: us-east-1 - role-duration-seconds: 900 - role-skip-session-tagging: true - name: Update Nightly Distro Downloads page run: | dist_path="snapshot/STS4/nightly/dist/e4.28" - s3_url=s3://tools-spring-io/${dist_path}/ + s3_url=s3://$AWS_S3_BUCKET/${dist_path}/ downloads_html="sts4-nightly-e4.28.html" files=`aws s3 cp ${s3_url} . --recursive --exclude "*" --include "spring-tool-suite-4*.zip" --include "spring-tool-suite-4*.dmg" --include "spring-tool-suite-4*.self-extracting.jar" --include "spring-tool-suite-4*.tar.gz" --exclude "*/*" --dryrun` echo '' >> $downloads_html cat ./$downloads_html - aws s3 mv ./$downloads_html s3://tools-spring-io/snapshot/STS4/ --dryrun + aws s3 mv ./$downloads_html s3://$AWS_S3_BUCKET/snapshot/STS4/ --dryrun files=`aws s3 ls ${s3_url}` for file in $files + do do echo "file: ${file}" url="${s3_url}${file}" echo "url: ${url}" - if [[ "$file" =~ ^"s3://tools-spring-io" ]]; then - download_url=https://cdn.spring.io/spring-tools${file:20} + if [[ "$file" =~ ^"s3://$AWS_S3_BUCKET" ]]; then + download_url=${DOWNLOAD_URL_ROOT}${file:20} filename=${file:${#s3_url}} echo '
  • '${filename}'
    • ' >> $downloads_html fi diff --git a/.github/workflows/gh-hosted-eclipse-distro-build.yml b/.github/workflows/gh-hosted-eclipse-distro-build.yml index 8d6ee5b15a..8fcd477292 100644 --- a/.github/workflows/gh-hosted-eclipse-distro-build.yml +++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml @@ -27,6 +27,14 @@ on: default: '17' type: string +env: + AWS_ACCESS_KEY_ID: ${{ secrets.CDN_S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CDN_S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT_URL_S3: ${{ secrets.CDN_S3_ENDPOINT }} + AWS_S3_BUCKET: ${{ secrets.CDN_BUCKET }} + DOWNLOAD_URL_ROOT: https://cdn-test.spring.io/spring-tools + jobs: eclipse-distro-build: @@ -85,24 +93,13 @@ jobs: p2_path=`cat ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/s3-p2-repo-dist-path.txt` echo "dist_path=$dist_path" >> $GITHUB_OUTPUT echo "p2_path=$p2_path" >> $GITHUB_OUTPUT - - name: Cloudgate S3 Configuration - if: always() - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a - with: - aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} - role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} - role-session-name: ${{ github.run_id }} - aws-region: us-east-1 - role-duration-seconds: 900 - role-skip-session-tagging: true - name: Upload P2 Repo Build Artifacts run: | cd eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target p2_path=${{ steps.s3-paths.outputs.p2_path }} echo "P2 path: ${p2_path}" - aws s3 rm s3://tools-spring-io/$p2_path/ --recursive - aws s3 cp ./repository/ s3://tools-spring-io/$p2_path/ --recursive --no-progress + aws s3 rm s3://$AWS_S3_BUCKET/$p2_path/ --recursive + aws s3 cp ./repository/ s3://$AWS_S3_BUCKET/$p2_path/ --recursive --no-progress - name: Upload Linux Distro Build Artifacts run: | cd eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products @@ -125,9 +122,9 @@ jobs: dist_path=${{ steps.s3-paths.outputs.dist_path }} ls spring-tool-suite-4*linux*.tar.gz* echo "Removing old Linux .tar.gz files from Akamai s3..." - aws s3 rm s3://tools-spring-io/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" + aws s3 rm s3://$AWS_S3_BUCKET/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" echo "Uploading new Linux .tar.gz files to Akamai s3..." - aws s3 cp . s3://tools-spring-io/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --no-progress + aws s3 cp . s3://$AWS_S3_BUCKET/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --no-progress - name: Update Nightly Distro Downloads page if: ${{ inputs.build_type == 'snapshot' }} run: | @@ -138,7 +135,7 @@ jobs: run: | echo "Uploading Win Zips and OSX tar.gz to S3 for signing..." id=${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }} - aws s3 cp ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products s3://tools-spring-io/sts4-distro-ci-temp/$id --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress + aws s3 cp ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products s3://$AWS_S3_BUCKET/sts4-distro-ci-temp/$id --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress outputs: id: ${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }} dist_path: ${{ steps.s3-paths.outputs.dist_path }} @@ -153,22 +150,12 @@ jobs: with: sparse-checkout: | .github - - name: Cloudgate S3 Configuration - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a - with: - aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} - role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} - role-session-name: ${{ github.run_id }} - aws-region: us-east-1 - role-duration-seconds: 3600 - role-skip-session-tagging: true - name: Download Win Zips from S3 for Signing run: | rm -f spring-tool-suite-4*win*.zip* rm -f spring-tool-suite-4*win*.self-extracting.jar* ls - aws s3 mv s3://tools-spring-io/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --exclude "*/*" + aws s3 mv s3://$AWS_S3_BUCKET/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --exclude "*/*" - name: Sign EXE within zip files id: sign env: @@ -188,9 +175,9 @@ jobs: ls spring-tool-suite-4*win*.zip* ls spring-tool-suite-4*win*.self-extracting.jar* echo "Removing old win zip and self extracting jar files from Akamai..." - aws s3 rm s3://tools-spring-io/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*" + aws s3 rm s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*" echo "Uploading new win zip and self extracting jar files to Akamai..." - aws s3 mv . s3://tools-spring-io/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*" --no-progress + aws s3 mv . s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*" --no-progress - name: Update Nightly Distro Downloads page if: ${{ inputs.build_type == 'snapshot' }} run: | @@ -238,21 +225,11 @@ jobs: - name: Install appdmg run: | npm install -g appdmg - - name: Cloudgate S3 Configuration - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a - with: - aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} - role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} - role-session-name: ${{ github.run_id }} - aws-region: us-east-1 - role-duration-seconds: 900 - role-skip-session-tagging: true - name: Download OSX tar.gz files from S3 for Signing/Notarization run: | rm -rf *macosx* ls - aws s3 mv s3://tools-spring-io/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress + aws s3 mv s3://$AWS_S3_BUCKET/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress - name: Sign .app, Create and Sign DMG env: MACOS_CERTIFICATE_ID: ${{ secrets.MACOS_CERTIFICATE_ID }} @@ -287,16 +264,6 @@ jobs: echo "Failed Notarization" exit 1 fi - - name: Cloudgate S3 Configuration - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a - with: - aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} - role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} - role-session-name: ${{ github.run_id }}_upload - aws-region: us-east-1 - role-duration-seconds: 900 - role-skip-session-tagging: true - name: Update DMG files on S3 id: update-s3 run: | @@ -304,9 +271,9 @@ jobs: echo "Processing S3 update..." ls spring-tool-suite-4*macosx*.dmg* echo "Removing old dmg files from Akamai..." - aws s3 rm s3://tools-spring-io/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*" + aws s3 rm s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*" echo "Uploading new dmg files to Akamai..." - aws s3 mv . s3://tools-spring-io/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*" --no-progress + aws s3 mv . s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*" --no-progress - name: Update Nightly Distro Downloads page if: ${{ inputs.build_type == 'snapshot' && always() }} run: | @@ -328,21 +295,11 @@ jobs: with: sparse-checkout: | .github - - name: Cloudgate S3 Configuration - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a - with: - aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} - role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} - role-session-name: ${{ github.run_id }} - aws-region: us-east-1 - role-duration-seconds: 900 - role-skip-session-tagging: true - name: Collect URLs on Akamai to Purge id: collect-urls run: | - dist_urls=`aws s3 ls s3://tools-spring-io/${{ needs.eclipse-distro-build.outputs.dist_path }} --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -` - p2_urls=`aws s3 ls s3://tools-spring-io/${{ needs.eclipse-distro-build.outputs.p2_path }} --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -` + dist_urls=`aws s3 ls s3://$AWS_S3_BUCKET/${{ needs.eclipse-distro-build.outputs.dist_path }} --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -` + p2_urls=`aws s3 ls s3://$AWS_S3_BUCKET/${{ needs.eclipse-distro-build.outputs.p2_path }} --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -` urls="${dist_urls} ${p2_urls}" echo "urls=$urls" >> $GITHUB_OUTPUT - uses: ./.github/actions/akamai-purge-osx @@ -362,20 +319,10 @@ jobs: with: sparse-checkout: | .github - - name: Cloudgate S3 Configuration - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a - with: - aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} - role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} - role-session-name: ${{ github.run_id }} - aws-region: us-east-1 - role-duration-seconds: 900 - role-skip-session-tagging: true - name: Remove Temp Build Artifacts from S3 id: cleanup-s3-temp-storage run: | - aws s3 rm s3://tools-spring-io/sts4-distro-ci-temp --recursive --exclude "*" --include "${{ needs.eclipse-distro-build.outputs.id }}/*" + aws s3 rm s3://$AWS_S3_BUCKET/sts4-distro-ci-temp --recursive --exclude "*" --include "${{ needs.eclipse-distro-build.outputs.id }}/*" notify-failure: needs: [ eclipse-distro-build, sign-win-distros, sign-osx-distros, cleanup, purge_cache ] diff --git a/.github/workflows/snapshot-eclipse-ls-extensions-build.yml b/.github/workflows/snapshot-eclipse-ls-extensions-build.yml index 299dbcecea..34be0ef97b 100644 --- a/.github/workflows/snapshot-eclipse-ls-extensions-build.yml +++ b/.github/workflows/snapshot-eclipse-ls-extensions-build.yml @@ -16,4 +16,5 @@ jobs: with: eclipse_profile: 'e431' build_type: 'snapshot' + skip_tests: true secrets: inherit