From 30cf55be05fb629340f0a6916c8050226d071884 Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Mon, 28 Aug 2023 22:40:04 +0530 Subject: [PATCH 1/5] Chore:Switching to Service Account JSON Credentials for GA Packs Download from Google Drive during migration automation Description: This pull request introduces a pivotal enhancement to the GitHub environment's mechanism for downloading GA packs from Google Drive. Changes Made: Transition to Service Account JSON Credentials: The method for accessing Google Drive has been revamped. Instead of using a GCP API key, we now leverage a Service Account to download files. This is a more secure approach to handle file downloads from Google Drive. For this purpose, a dedicated file (creds.json) has been created to securely store the Service Account JSON credentials. This file is located within the .github/migration-tester/migration-automation directory. Removal of Old GCP Credentials: The older method of using the GCP API key, represented by the secrets GCP_CLIENT_ID, GCP_CLIENT_SECRET, and GCP_REFRESH_TOKEN, has been deprecated and removed from the workflow. This change aligns with the shift towards utilizing the Service Account for the file download operations. --- .github/workflows/migration-automation.yml | 29 ++++++++++++++-------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/migration-automation.yml b/.github/workflows/migration-automation.yml index b5827e9d05..bdb76e17f9 100644 --- a/.github/workflows/migration-automation.yml +++ b/.github/workflows/migration-automation.yml @@ -54,6 +54,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + - name: Save service account JSON credentials to a file + run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - name: Setup Docker environment run: | # Get the ID of the workflow from the GitHub API using curl and jq @@ -114,7 +116,7 @@ jobs: - name: Execute Migration Automation Script Ubuntu run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" "${{ secrets.GCP_CLIENT_ID }}" "${{ secrets.GCP_CLIENT_SECRET }}" "${{ secrets.GCP_REFRESH_TOKEN }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - name: Persist Logs @@ -137,10 +139,12 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + - name: Save service account JSON credentials to a file + run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - name: Execute Migration Automation Script Ubuntu run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" "${{ secrets.GCP_CLIENT_ID }}" "${{ secrets.GCP_CLIENT_SECRET }}" "${{ secrets.GCP_REFRESH_TOKEN }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - name: Persist Logs @@ -173,6 +177,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + - name: Save service account JSON credentials to a file + run: echo '${{ secrets.JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - name: Create MS SQL Database run: | @@ -194,8 +200,7 @@ jobs: - name: Execute Migration Automation Script Ubuntu run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" "${{ secrets.GCP_CLIENT_ID }}" "${{ secrets.GCP_CLIENT_SECRET }}" "${{ secrets.GCP_REFRESH_TOKEN }}" | tee ${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt - continue-on-error: true + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" - name: Persist Logs run: | @@ -216,11 +221,13 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + - name: Save service account JSON credentials to a file + run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - name: Execute Migration Automation Script Mac run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" "${{ secrets.GCP_CLIENT_ID }}" "${{ secrets.GCP_CLIENT_SECRET }}" "${{ secrets.GCP_REFRESH_TOKEN }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - name: Persist Logs run: | @@ -241,6 +248,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + - name: Save service account JSON credentials to a file + run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - name: Set up PostgreSQL env: PGDATA: /usr/local/var/postgres @@ -267,8 +276,7 @@ jobs: - name: Execute Migration Automation Script Mac run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" "${{ secrets.GCP_CLIENT_ID }}" "${{ secrets.GCP_CLIENT_SECRET }}" "${{ secrets.GCP_REFRESH_TOKEN }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" - continue-on-error: true + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" - name: Persist Logs run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts @@ -288,6 +296,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + - name: Save service account JSON credentials to a file + run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - uses: potatoqualitee/mssqlsuite@v1.7 with: @@ -315,10 +325,7 @@ jobs: - name: Execute Migration Automation Script Mac run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" "${{ secrets.GCP_CLIENT_ID }}" "${{ secrets.GCP_CLIENT_SECRET }}" "${{ secrets.GCP_REFRESH_TOKEN }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" - continue-on-error: true - - name: Persist Logs - run: | + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts cp ${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts/ if: ${{ always() }} From ebda6fd08a406dcfd2410cbc097c69b43244564a Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Mon, 28 Aug 2023 23:10:51 +0530 Subject: [PATCH 2/5] Fix: continue-on-error: true made continue-on-error value to true. --- .github/workflows/migration-automation.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/migration-automation.yml b/.github/workflows/migration-automation.yml index bdb76e17f9..e8f8051267 100644 --- a/.github/workflows/migration-automation.yml +++ b/.github/workflows/migration-automation.yml @@ -201,7 +201,7 @@ jobs: run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" - + continue-on-error: true - name: Persist Logs run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts @@ -277,6 +277,7 @@ jobs: run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + continue-on-error: true - name: Persist Logs run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts @@ -326,6 +327,9 @@ jobs: run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + continue-on-error: true + - name: Persist Logs + run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts cp ${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts/ if: ${{ always() }} From c08b4539121351c1d5c19555c354972fb4bfd25c Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Wed, 30 Aug 2023 23:35:46 +0530 Subject: [PATCH 3/5] fix: Removed creds.json Ensured secure removal of creds.json from .github/workflows. --- .github/workflows/migration-automation.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/migration-automation.yml b/.github/workflows/migration-automation.yml index e8f8051267..fb234e3f45 100644 --- a/.github/workflows/migration-automation.yml +++ b/.github/workflows/migration-automation.yml @@ -118,6 +118,9 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true + - name: Remove creds.json file + run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Persist Logs run: | @@ -146,6 +149,9 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true + - name: Remove creds.json file + run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Persist Logs run: | @@ -202,6 +208,9 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true + - name: Remove creds.json file + run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Persist Logs run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts @@ -229,6 +238,9 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true + - name: Remove creds.json file + run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Persist Logs run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts @@ -278,6 +290,9 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true + - name: Remove creds.json file + run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Persist Logs run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts @@ -328,6 +343,9 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true + - name: Remove creds.json file + run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Persist Logs run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts From 1b056c8ff593de95e73bddb768b9da10a33306a3 Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Thu, 14 Sep 2023 17:16:20 +0530 Subject: [PATCH 4/5] refactor: Secure Token Handling and Streamline Script Enhanced security by decoding sensitive JSON credentials directly from the environment variable and generating a JWT token for authentication. This eliminates the need to store credentials in a file.(creds.json) --- .github/workflows/migration-automation.yml | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/migration-automation.yml b/.github/workflows/migration-automation.yml index fb234e3f45..98e0f97632 100644 --- a/.github/workflows/migration-automation.yml +++ b/.github/workflows/migration-automation.yml @@ -54,8 +54,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Save service account JSON credentials to a file - run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Set JSON credentials as an environment variable + run: echo "MIGRATION_JSON_ENCODED=$(echo '${{ secrets.MIGRATION_JSON }}' | base64 -w0)" >> $GITHUB_ENV - name: Setup Docker environment run: | # Get the ID of the workflow from the GitHub API using curl and jq @@ -116,7 +116,7 @@ jobs: - name: Execute Migration Automation Script Ubuntu run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - name: Remove creds.json file run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json @@ -142,12 +142,12 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Save service account JSON credentials to a file - run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Set JSON credentials as an environment variable + run: echo "MIGRATION_JSON_ENCODED=$(echo '${{ secrets.MIGRATION_JSON }}' | base64 -w0)" >> $GITHUB_ENV - name: Execute Migration Automation Script Ubuntu run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - name: Remove creds.json file run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json @@ -183,8 +183,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 - - name: Save service account JSON credentials to a file - run: echo '${{ secrets.JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Set JSON credentials as an environment variable + run: echo "MIGRATION_JSON_ENCODED=$(echo '${{ secrets.MIGRATION_JSON }}' | base64 -w0)" >> $GITHUB_ENV - name: Create MS SQL Database run: | @@ -206,7 +206,7 @@ jobs: - name: Execute Migration Automation Script Ubuntu run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - name: Remove creds.json file run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json @@ -230,13 +230,13 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Save service account JSON credentials to a file - run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Set JSON credentials as an environment variable + run: echo "MIGRATION_JSON_ENCODED=$(echo '${{ secrets.MIGRATION_JSON }}' | base64 -w0)" >> $GITHUB_ENV - name: Execute Migration Automation Script Mac run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - name: Remove creds.json file run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json @@ -260,8 +260,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Save service account JSON credentials to a file - run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Set JSON credentials as an environment variable + run: echo "MIGRATION_JSON_ENCODED=$(echo '${{ secrets.MIGRATION_JSON }}' | base64 -w0)" >> $GITHUB_ENV - name: Set up PostgreSQL env: PGDATA: /usr/local/var/postgres @@ -288,7 +288,7 @@ jobs: - name: Execute Migration Automation Script Mac run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - name: Remove creds.json file run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json @@ -312,8 +312,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Save service account JSON credentials to a file - run: echo '${{ secrets.MIGRATION_JSON }}' > ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json + - name: Set JSON credentials as an environment variable + run: echo "MIGRATION_JSON_ENCODED=$(echo '${{ secrets.MIGRATION_JSON }}' | base64 -w0)" >> $GITHUB_ENV - uses: potatoqualitee/mssqlsuite@v1.7 with: @@ -341,7 +341,7 @@ jobs: - name: Execute Migration Automation Script Mac run: | chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh - sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" creds.json | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" + sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - name: Remove creds.json file run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json From 3db1330fe3565d54e7101b457a9d2e529d6a229d Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Thu, 14 Sep 2023 17:25:14 +0530 Subject: [PATCH 5/5] remove: creds.json The removal of the creds.json file was executed because this step has been superseded by the introduction of environment variables when using JSON credentials. This change prevents the storage of JSON credentials within a file, enhancing security. --- .github/workflows/migration-automation.yml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/migration-automation.yml b/.github/workflows/migration-automation.yml index 98e0f97632..3f433ad49d 100644 --- a/.github/workflows/migration-automation.yml +++ b/.github/workflows/migration-automation.yml @@ -118,10 +118,7 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - - name: Remove creds.json file - run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - - + - name: Persist Logs run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts @@ -149,10 +146,7 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - - name: Remove creds.json file - run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - - + - name: Persist Logs run: | mkdir -p ${{ github.workspace }}/.github/migration-tester/migration-automation/artifacts @@ -208,8 +202,6 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/ubuntu-os/migration-script-ubuntu.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - - name: Remove creds.json file - run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - name: Persist Logs run: | @@ -238,8 +230,6 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - - name: Remove creds.json file - run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - name: Persist Logs run: | @@ -290,8 +280,6 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - - name: Remove creds.json file - run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - name: Persist Logs run: | @@ -343,8 +331,6 @@ jobs: chmod +x ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh sh ${{ github.workspace }}/.github/migration-tester/migration-automation/mac-os/migration-script-mac.sh "${{ github.event.inputs.urlOld }}" "${{ github.event.inputs.urlNew }}" "${{ github.event.inputs.currentVersion }}" "${{ github.event.inputs.migratingVersion }}" "${{ github.event.inputs.database }}" "${{ github.event.inputs.os }}" "${{ secrets.MIGRATION_EMAIL }}" "${{ secrets.MIGRATION_PASSWORD }}" "${{ secrets.MIGRATION_PAT }}" | tee "${{ github.workspace }}/.github/migration-tester/migration-automation/logs.txt" continue-on-error: true - - name: Remove creds.json file - run: rm -f ${{ github.workspace }}/.github/migration-tester/migration-automation/creds.json - name: Persist Logs run: |