Add GitHub backup procedure

.github/workflows/github-backup-procedure.yaml

@@ -0,0 +1,156 @@
+name: GitHub Backup Procedure
+
+on:
+  push:
+    branches:
+      - DO_NOT_TOUCH_THIS_BRANCH_FOR_BACKUP_PROCEDURE
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+# A workflows run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  get-environments:
+    # The type of runner that the job will run on
+    runs-on: ${{ vars.DGX_RUNNER }}
+    outputs:
+      names: ${{ steps.set_env.outputs.names }}
+    env:
+      repository_backup_path: /raid/ailab/workspace/aie_share_folder/emu-migration/${{ github.event.repository.name }}
+    steps:
+      - name: Get environments
+        uses: octokit/request-action@v2.3.0
+        id: get_env
+        with:
+          route: GET /repos/${{ github.repository }}/environments
+        env:
+          GITHUB_TOKEN: ${{ secrets.CICD_TOKEN }}
+      - name: Set output
+        id: set_env
+        run: |
+          echo "Remove old backup folder"
+          rm -rf $repository_backup_path
+          echo "Create backup folder"
+          mkdir -p $repository_backup_path
+          names=$(echo $NAMES | jq -c '[.[]]' -j)
+          echo $names
+          if [[ "[]" == "$names" ]]; then
+            echo "names=[\"empty\"]" >> $GITHUB_OUTPUT
+          else
+            echo "names=$names" >> $GITHUB_OUTPUT
+          fi
+          touch $repository_backup_path/started.txt
+          echo "Check if environment with protection rules exists"
+          export ENV_SECRETS=$(curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{secrets.CICD_TOKEN}}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/AILAB-bh/${{ github.event.repository.name }}/environments)
+          result=( $(echo "$ENV_SECRETS" | jq -r '.environments[] | select(.protection_rules != []) | .name') )
+          # loop result
+          for environment in "${result[@]}"; do
+              curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{secrets.CICD_TOKEN}}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/AILAB-bh/${{ github.event.repository.name }}/environments/$environment/deployment-branch-policies -d '{"name":"DO_NOT_TOUCH_THIS_BRANCH_FOR_BACKUP_PROCEDURE", "type":"branch"}'
+          done
+        env:
+          NAMES: ${{ toJson(fromJson(steps.get_env.outputs.data).environments.*.name) }}
+
+  extract-secrets-variable-for-the-environment:
+    needs: get-environments
+    continue-on-error: true
+    # The type of runner that the job will run on
+    runs-on: ${{ vars.DGX_RUNNER }}
+    environment:
+      name: ${{ matrix.environment }}
+    env:
+      repository_backup_path: /raid/ailab/workspace/aie_share_folder/emu-migration/${{ github.event.repository.name }}
+      ORG_REPO_ENV_SECRETS: '${{ toJson(secrets) }}'
+      ORG_REPO_ENV_VARIABLES: '${{ toJson(vars) }}'
+    strategy:
+      matrix:
+        environment: ${{ fromJSON(needs.get-environments.outputs.names) }}
+    steps:
+      - run: |
+          if [[ "${{ matrix.environment }}" == "empty" ]]; then
+            echo "No environment to backup"
+          else
+            echo "Extract secrets for environment: ${{ matrix.environment }}"
+            export ENV_SECRETS=$(curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{secrets.CICD_TOKEN}}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/AILAB-bh/${{ github.event.repository.name }}/environments/${{ matrix.environment }}/secrets?per_page=100&page=1)
+            result=( $(echo "$ORG_REPO_ENV_SECRETS" | jq -r 'keys[]') )
+            for secret in "${result[@]}"; do
+              # check if secret if present into the ENV_SECRETS
+              if [[ "$ENV_SECRETS" =~ .*"$secret".* ]]; then
+                echo "Secret: $secret found"
+                value="$(jq -r --arg key "$secret" '.[$key]' <<< "$ORG_REPO_ENV_SECRETS" | sed 's|.|&|g')"
+                echo "$value" >> $repository_backup_path/SECRET@${{ github.event.repository.name }}@${{ matrix.environment }}@$secret.backup
+              fi              
+            done
+          
+            echo "Extract variables for environment: ${{ matrix.environment }}"
+            export ENV_VARIABLES=$(curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{secrets.CICD_TOKEN}}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/AILAB-bh/${{ github.event.repository.name }}/environments/${{ matrix.environment }}/variables?per_page=100&page=1)
+            result=( $(echo "$ORG_REPO_ENV_VARIABLES" | jq -r 'keys[]') )
+            for variable in "${result[@]}"; do
+              # check if secret if present into the ENV_VARIABLES
+              if [[ "$ENV_VARIABLES" =~ .*"$variable".* ]]; then
+                echo "Variable: $variable found"
+                value="$(jq -r --arg key "$variable" '.[$key]' <<< "$ORG_REPO_ENV_VARIABLES" | sed 's|.|&|g')"
+                echo "$value" >> $repository_backup_path/VARIABLE@${{ github.event.repository.name }}@${{ matrix.environment }}@$variable.backup
+              fi            
+            done
+          fi
+
+  extract-secrets-variable-for-the-repo:
+    needs: extract-secrets-variable-for-the-environment
+    # The type of runner that the job will run on
+    runs-on: ${{ vars.DGX_RUNNER }}
+    env:
+      repository_backup_path: /raid/ailab/workspace/aie_share_folder/emu-migration/${{ github.event.repository.name }}
+      ORG_REPO_SECRETS: '${{ toJson(secrets) }}'
+      ORG_REPO_VARIABLES: '${{ toJson(vars) }}'
+    steps:
+      - run: |
+          echo "Extract secrets for repo: ${{ github.event.repository.name }}"
+          export REPO_SECRETS=$(curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{secrets.CICD_TOKEN}}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/AILAB-bh/${{ github.event.repository.name }}/actions/secrets?per_page=100&page=1)
+          result=( $(echo "$ORG_REPO_SECRETS" | jq -r 'keys[]') )
+          for secret in "${result[@]}"; do
+            # check if secret if present into the REPO_SECRETS
+            if [[ "$REPO_SECRETS" =~ .*"$secret".* ]]; then
+              echo "Secret: $secret found"
+              value="$(jq -r --arg key "$secret" '.[$key]' <<< "$ORG_REPO_SECRETS" | sed 's|.|&|g')"
+              echo "$value" >> $repository_backup_path/SECRET@${{ github.event.repository.name }}@$secret.backup
+            fi              
+          done
+
+          echo "Extract variables for repo: ${{ github.event.repository.name }}"
+          export REPO_VARIABLES=$(curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{secrets.CICD_TOKEN}}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/AILAB-bh/${{ github.event.repository.name }}/actions/variables?per_page=100&page=1)
+          result=( $(echo "$ORG_REPO_VARIABLES" | jq -r 'keys[]') )
+          for variable in "${result[@]}"; do
+            # check if secret if present into the REPO_VARIABLES
+            if [[ "$REPO_VARIABLES" =~ .*"$variable".* ]]; then
+              echo "Variable: $variable found"
+              value="$(jq -r --arg key "$variable" '.[$key]' <<< "$ORG_REPO_VARIABLES" | sed 's|.|&|g')"
+              echo "$value" >> $repository_backup_path/VARIABLE@${{ github.event.repository.name }}@$variable.backup
+            fi            
+          done
+
+  backup-secrets-variable-for-the-repo:
+    needs: extract-secrets-variable-for-the-repo
+    # The type of runner that the job will run on
+    runs-on: ${{ vars.DGX_RUNNER }}
+    env:
+      repository_backup_path: /raid/ailab/workspace/aie_share_folder/emu-migration/${{ github.event.repository.name }}
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v4
+        with:
+          ref: DO_NOT_TOUCH_THIS_BRANCH_FOR_BACKUP_PROCEDURE
+
+      - run: |
+          echo "Remove old backup folder"
+          rm -f $repository_backup_path/started.txt
+          touch $repository_backup_path/completed.txt
+          echo "Zip all the backup files"
+          zip -P ${{ secrets.PYPY_PASSWORD }} $repository_backup_path/${{ github.event.repository.name }}.zip $repository_backup_path/*.backup
+          
+          echo "Run python script to create result json"
+          touch ${{ github.event.repository.name }}.json
+          python src/create_json_from_backup_folder.py $repository_backup_path ${{ github.event.repository.name }}
+          
+          echo "Save data to mlflow"
+          curl --location 'https://tps-innovation-dev.np-0000111.npaeuw1.bakerhughes.com/ailab-mlops-toolkit-sp/mlflow_experiment' --header 'accept: application/json' --form 'data="{\"experiment_name\":\"${{github.event.repository.name}}\",\"metrics\":{\"m1\":1,\"m2\":2},\"params\":{\"p1\":\"foo\",\"p2\":\"faa\"}}"' --form 'files=@"/raid/ailab/workspace/aie_share_folder/emu-migration/${{ github.event.repository.name }}/${{ github.event.repository.name }}.zip"'

src/create_json_from_backup_folder.py

@@ -0,0 +1,79 @@
+import json
+import os
+import sys
+
+BACKUP_FILE_EXTENSION = ".backup"
+
+
+def create_json_from_backup_folder(repo_backup_folder: str, repo_name: str):
+    """
+    This function creates a json file from the backup folder
+    """
+    secrets = []
+    variables = []
+    environments = {}
+    list_of_files = os.listdir(repo_backup_folder)
+    for file in list_of_files:
+        if file.endswith(BACKUP_FILE_EXTENSION):
+            # split the file name by @ and check the length of the split is 4
+            split_file_name = file.split("@")
+            file_type = split_file_name[0]
+            environment_name = None
+            element_name = None
+            if len(split_file_name) == 4:
+                environment_name = split_file_name[2]
+                element_name = split_file_name[3].replace(BACKUP_FILE_EXTENSION, "")
+            elif len(split_file_name) == 3:
+                element_name = split_file_name[2].replace(BACKUP_FILE_EXTENSION, "")
+            else:
+                print("Invalid file name")
+
+            with open(repo_backup_folder + "/" + file, 'r') as file:
+                file_content = file.read().replace("\n", "")
+
+            print(f"File type: {file_type}, Environment name: {environment_name}, Element name: {element_name}")
+            if environment_name is not None:
+                if environment_name not in environments:
+                    environments[environment_name] = {"secrets": [], "variables": []}
+                if file_type == "SECRET":
+                    environments[environment_name]["secrets"].append(element_name + ": " + file_content)
+                elif file_type == "VARIABLE":
+                    environments[environment_name]["variables"].append(element_name + ": " + file_content)
+            else:
+                if file_type == "SECRET":
+                    secrets.append(element_name + ": " + file_content)
+                elif file_type == "VARIABLE":
+                    variables.append(element_name + ": " + file_content)
+
+    # create json
+    d = {"secrets": "", "variables": ""}
+    d["secrets"] = {"repository": {}}
+    for sec in secrets:
+        key, val = sec.split(": ")
+        d["secrets"]["repository"][key] = val
+    d["variables"] = {"repository": {}}
+    for var in variables:
+        key, val = var.split(": ")
+        d["variables"]["repository"][key] = val
+
+    for env in environments:
+        d["secrets"][env] = {}
+        d["variables"][env] = {}
+        for sec in environments[env]["secrets"]:
+            key, val = sec.split(": ")
+            d["secrets"][env][key] = val
+        for var in environments[env]["variables"]:
+            key, val = var.split(": ")
+            d["variables"][env][key] = val
+
+    print(d)
+    # save json into the backup folder
+    with open(repo_backup_folder + "/" + repo_name + ".json", 'w') as file:
+        json.dump(d, file)
+
+
+backup_folder = sys.argv[1]
+repo_name = sys.argv[2]
+
+if __name__ == "__main__":
+    create_json_from_backup_folder(backup_folder, repo_name)