Skip to content

Commit

Permalink
[GHA] Eclipse LS extensions and distro snapshot to R2 bucket
Browse files Browse the repository at this point in the history
[GHA] Revert to main branch
  • Loading branch information
BoykoAlex committed Jul 10, 2024
1 parent c2bb548 commit d8a68ec
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 104 deletions.
23 changes: 10 additions & 13 deletions .github/workflows/eclipse-ls-extensions-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,13 @@ on:
description: version of sts4 ls extensions
required: false
type: string
env:
AWS_ACCESS_KEY_ID: ${{ secrets.CDN_S3_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CDN_S3_SECRET_KEY }}
AWS_DEFAULT_REGION: us-east-1
AWS_ENDPOINT_URL_S3: ${{ secrets.CDN_S3_ENDPOINT }}
AWS_S3_BUCKET: ${{ secrets.CDN_BUCKET }}
DOWNLOAD_URL_ROOT: https://cdn-test.spring.io/spring-tools

jobs:

Expand Down Expand Up @@ -74,24 +81,14 @@ jobs:
else
./mvnw --batch-mode clean install -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Dtycho.localArtifacts=ignore
fi
- name: Cloudgate S3 Configuration
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }}
role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
role-session-name: ${{ github.run_id }}
aws-region: us-east-1
role-duration-seconds: 900
role-skip-session-tagging: true
- name: Upload P2 Repo to S3 and Akamai
id: upload-p2-to-akamai
run: |
p2_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt`
echo "p2_path=$p2_path" >> $GITHUB_OUTPUT
echo "P2 path: ${p2_path}"
aws s3 rm s3://tools-spring-io/${p2_path}/ --recursive
aws s3 cp ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/ s3://tools-spring-io/${p2_path}/ --recursive --no-progress
aws s3 rm s3://$AWS_S3_BUCKET/${p2_path}/ --recursive
aws s3 cp ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/ s3://$AWS_S3_BUCKET/${p2_path}/ --recursive --no-progress
# - name: Verify Uploaded P2 Repo
# id: verify-p2-on-akamai
# run: |
Expand All @@ -106,7 +103,7 @@ jobs:
if: ${{ inputs.build_type != 'snapshot' }}
id: compute-invalid-urls
run: |
invalid_urls=`aws s3 ls s3://tools-spring-io/${{ steps.upload-p2-to-akamai.outputs.p2_path }}/ --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -`
invalid_urls=`aws s3 ls s3://$AWS_S3_BUCKET/${{ steps.upload-p2-to-akamai.outputs.p2_path }}/ --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -`
echo "invalid_urls=$invalid_urls" >> $GITHUB_OUTPUT
- name: Verify Eclipse LS Extensions on e433
if: ${{ inputs.build_type == 'snapshot' }}
Expand Down
30 changes: 14 additions & 16 deletions .github/workflows/gen-nighly-downloads.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,47 +3,45 @@ name: Generate Nightly Downloads HTML
# configure manual trigger
on:
workflow_dispatch:
env:
AWS_ACCESS_KEY_ID: ${{ secrets.CDN_S3_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CDN_S3_SECRET_KEY }}
AWS_DEFAULT_REGION: us-east-1
AWS_ENDPOINT_URL_S3: ${{ secrets.CDN_S3_ENDPOINT }}
AWS_S3_BUCKET: ${{ secrets.CDN_BUCKET }}
DOWNLOAD_URL_ROOT: https://cdn-test.spring.io/spring-tools

jobs:
gen_html:
runs-on: ubuntu-latest
steps:
- name: Cloudgate S3 Configuration
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }}
role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
role-session-name: ${{ github.run_id }}
aws-region: us-east-1
role-duration-seconds: 900
role-skip-session-tagging: true
- name: Update Nightly Distro Downloads page
run: |
dist_path="snapshot/STS4/nightly/dist/e4.28"
s3_url=s3://tools-spring-io/${dist_path}/
s3_url=s3://$AWS_S3_BUCKET/${dist_path}/
downloads_html="sts4-nightly-e4.28.html"
files=`aws s3 cp ${s3_url} . --recursive --exclude "*" --include "spring-tool-suite-4*.zip" --include "spring-tool-suite-4*.dmg" --include "spring-tool-suite-4*.self-extracting.jar" --include "spring-tool-suite-4*.tar.gz" --exclude "*/*" --dryrun`
echo '<ul>' >> $downloads_html
for file in $files
do
if [[ "$file" =~ ^"s3://tools-spring-io" ]]; then
download_url=https://cdn.spring.io/spring-tools${file:20}
if [[ "$file" =~ ^"s3://$AWS_S3_BUCKET" ]]; then
download_url=${DOWNLOAD_URL_ROOT}${file:20}
filename=${file:${#s3_url}}
echo ' <li><a href="'${download_url}'">'${filename}'</li>' >> $downloads_html
fi
done
echo '</ul>' >> $downloads_html
cat ./$downloads_html
aws s3 mv ./$downloads_html s3://tools-spring-io/snapshot/STS4/ --dryrun
aws s3 mv ./$downloads_html s3://$AWS_S3_BUCKET/snapshot/STS4/ --dryrun
files=`aws s3 ls ${s3_url}`
for file in $files
do
do
echo "file: ${file}"
url="${s3_url}${file}"
echo "url: ${url}"
if [[ "$file" =~ ^"s3://tools-spring-io" ]]; then
download_url=https://cdn.spring.io/spring-tools${file:20}
if [[ "$file" =~ ^"s3://$AWS_S3_BUCKET" ]]; then
download_url=${DOWNLOAD_URL_ROOT}${file:20}
filename=${file:${#s3_url}}
echo ' <li><a href="'${download_url}'">'${filename}'</li>' >> $downloads_html
fi
Expand Down
97 changes: 22 additions & 75 deletions .github/workflows/gh-hosted-eclipse-distro-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,14 @@ on:
default: '17'
type: string

env:
AWS_ACCESS_KEY_ID: ${{ secrets.CDN_S3_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CDN_S3_SECRET_KEY }}
AWS_DEFAULT_REGION: us-east-1
AWS_ENDPOINT_URL_S3: ${{ secrets.CDN_S3_ENDPOINT }}
AWS_S3_BUCKET: ${{ secrets.CDN_BUCKET }}
DOWNLOAD_URL_ROOT: https://cdn-test.spring.io/spring-tools

jobs:

eclipse-distro-build:
Expand Down Expand Up @@ -85,24 +93,13 @@ jobs:
p2_path=`cat ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/s3-p2-repo-dist-path.txt`
echo "dist_path=$dist_path" >> $GITHUB_OUTPUT
echo "p2_path=$p2_path" >> $GITHUB_OUTPUT
- name: Cloudgate S3 Configuration
if: always()
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }}
role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
role-session-name: ${{ github.run_id }}
aws-region: us-east-1
role-duration-seconds: 900
role-skip-session-tagging: true
- name: Upload P2 Repo Build Artifacts
run: |
cd eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target
p2_path=${{ steps.s3-paths.outputs.p2_path }}
echo "P2 path: ${p2_path}"
aws s3 rm s3://tools-spring-io/$p2_path/ --recursive
aws s3 cp ./repository/ s3://tools-spring-io/$p2_path/ --recursive --no-progress
aws s3 rm s3://$AWS_S3_BUCKET/$p2_path/ --recursive
aws s3 cp ./repository/ s3://$AWS_S3_BUCKET/$p2_path/ --recursive --no-progress
- name: Upload Linux Distro Build Artifacts
run: |
cd eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products
Expand All @@ -125,9 +122,9 @@ jobs:
dist_path=${{ steps.s3-paths.outputs.dist_path }}
ls spring-tool-suite-4*linux*.tar.gz*
echo "Removing old Linux .tar.gz files from Akamai s3..."
aws s3 rm s3://tools-spring-io/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*"
aws s3 rm s3://$AWS_S3_BUCKET/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*"
echo "Uploading new Linux .tar.gz files to Akamai s3..."
aws s3 cp . s3://tools-spring-io/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --no-progress
aws s3 cp . s3://$AWS_S3_BUCKET/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --no-progress
- name: Update Nightly Distro Downloads page
if: ${{ inputs.build_type == 'snapshot' }}
run: |
Expand All @@ -138,7 +135,7 @@ jobs:
run: |
echo "Uploading Win Zips and OSX tar.gz to S3 for signing..."
id=${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }}
aws s3 cp ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products s3://tools-spring-io/sts4-distro-ci-temp/$id --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress
aws s3 cp ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products s3://$AWS_S3_BUCKET/sts4-distro-ci-temp/$id --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress
outputs:
id: ${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }}
dist_path: ${{ steps.s3-paths.outputs.dist_path }}
Expand All @@ -153,22 +150,12 @@ jobs:
with:
sparse-checkout: |
.github
- name: Cloudgate S3 Configuration
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }}
role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
role-session-name: ${{ github.run_id }}
aws-region: us-east-1
role-duration-seconds: 3600
role-skip-session-tagging: true
- name: Download Win Zips from S3 for Signing
run: |
rm -f spring-tool-suite-4*win*.zip*
rm -f spring-tool-suite-4*win*.self-extracting.jar*
ls
aws s3 mv s3://tools-spring-io/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --exclude "*/*"
aws s3 mv s3://$AWS_S3_BUCKET/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --exclude "*/*"
- name: Sign EXE within zip files
id: sign
env:
Expand All @@ -188,9 +175,9 @@ jobs:
ls spring-tool-suite-4*win*.zip*
ls spring-tool-suite-4*win*.self-extracting.jar*
echo "Removing old win zip and self extracting jar files from Akamai..."
aws s3 rm s3://tools-spring-io/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*"
aws s3 rm s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*"
echo "Uploading new win zip and self extracting jar files to Akamai..."
aws s3 mv . s3://tools-spring-io/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*" --no-progress
aws s3 mv . s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*" --no-progress
- name: Update Nightly Distro Downloads page
if: ${{ inputs.build_type == 'snapshot' }}
run: |
Expand Down Expand Up @@ -238,21 +225,11 @@ jobs:
- name: Install appdmg
run: |
npm install -g appdmg
- name: Cloudgate S3 Configuration
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }}
role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
role-session-name: ${{ github.run_id }}
aws-region: us-east-1
role-duration-seconds: 900
role-skip-session-tagging: true
- name: Download OSX tar.gz files from S3 for Signing/Notarization
run: |
rm -rf *macosx*
ls
aws s3 mv s3://tools-spring-io/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress
aws s3 mv s3://$AWS_S3_BUCKET/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress
- name: Sign .app, Create and Sign DMG
env:
MACOS_CERTIFICATE_ID: ${{ secrets.MACOS_CERTIFICATE_ID }}
Expand Down Expand Up @@ -287,26 +264,16 @@ jobs:
echo "Failed Notarization"
exit 1
fi
- name: Cloudgate S3 Configuration
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }}
role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
role-session-name: ${{ github.run_id }}_upload
aws-region: us-east-1
role-duration-seconds: 900
role-skip-session-tagging: true
- name: Update DMG files on S3
id: update-s3
run: |
dist_path=${{ needs.eclipse-distro-build.outputs.dist_path }}
echo "Processing S3 update..."
ls spring-tool-suite-4*macosx*.dmg*
echo "Removing old dmg files from Akamai..."
aws s3 rm s3://tools-spring-io/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*"
aws s3 rm s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*"
echo "Uploading new dmg files to Akamai..."
aws s3 mv . s3://tools-spring-io/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*" --no-progress
aws s3 mv . s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*" --no-progress
- name: Update Nightly Distro Downloads page
if: ${{ inputs.build_type == 'snapshot' && always() }}
run: |
Expand All @@ -328,21 +295,11 @@ jobs:
with:
sparse-checkout: |
.github
- name: Cloudgate S3 Configuration
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }}
role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
role-session-name: ${{ github.run_id }}
aws-region: us-east-1
role-duration-seconds: 900
role-skip-session-tagging: true
- name: Collect URLs on Akamai to Purge
id: collect-urls
run: |
dist_urls=`aws s3 ls s3://tools-spring-io/${{ needs.eclipse-distro-build.outputs.dist_path }} --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -`
p2_urls=`aws s3 ls s3://tools-spring-io/${{ needs.eclipse-distro-build.outputs.p2_path }} --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -`
dist_urls=`aws s3 ls s3://$AWS_S3_BUCKET/${{ needs.eclipse-distro-build.outputs.dist_path }} --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -`
p2_urls=`aws s3 ls s3://$AWS_S3_BUCKET/${{ needs.eclipse-distro-build.outputs.p2_path }} --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -`
urls="${dist_urls} ${p2_urls}"
echo "urls=$urls" >> $GITHUB_OUTPUT
- uses: ./.github/actions/akamai-purge-osx
Expand All @@ -362,20 +319,10 @@ jobs:
with:
sparse-checkout: |
.github
- name: Cloudgate S3 Configuration
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }}
role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
role-session-name: ${{ github.run_id }}
aws-region: us-east-1
role-duration-seconds: 900
role-skip-session-tagging: true
- name: Remove Temp Build Artifacts from S3
id: cleanup-s3-temp-storage
run: |
aws s3 rm s3://tools-spring-io/sts4-distro-ci-temp --recursive --exclude "*" --include "${{ needs.eclipse-distro-build.outputs.id }}/*"
aws s3 rm s3://$AWS_S3_BUCKET/sts4-distro-ci-temp --recursive --exclude "*" --include "${{ needs.eclipse-distro-build.outputs.id }}/*"
notify-failure:
needs: [ eclipse-distro-build, sign-win-distros, sign-osx-distros, cleanup, purge_cache ]
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/snapshot-eclipse-ls-extensions-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,5 @@ jobs:
with:
eclipse_profile: 'e431'
build_type: 'snapshot'
skip_tests: true
secrets: inherit

0 comments on commit d8a68ec

Please sign in to comment.