Merge branch 'release-5.2.x'

opencb · Sep 13, 2024 · 89ee779 · 89ee779
2 parents b0812f6 + 113c1f2
commit 89ee779
Show file tree

Hide file tree

Showing 12 changed files with 205 additions and 47 deletions.
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -10,12 +10,7 @@ jobs:
   build:
     uses: ./.github/workflows/build-java-app-workflow.yml
 
-  test:
-    uses: ./.github/workflows/test-analysis.yml
-    needs: build
-    secrets: inherit
-
   deploy-maven:
     uses: ./.github/workflows/deploy-maven-repository-workflow.yml
-    needs: test
+    needs: build
     secrets: inherit
diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml
@@ -6,11 +6,36 @@ on:
     types: [ submitted ]
 
 jobs:
-  build:
-    uses: ./.github/workflows/build-java-app-workflow.yml
-
+  calculate-xetabase-branch:
+    name: Calculate Xetabase branch
+    runs-on: ubuntu-22.04
+    outputs:
+      xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }}
+    steps:
+      - name: Clone java-common-libs
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found"
+          persist-credentials: false
+      - id: get_xetabase_branch
+        name: "Get current branch for Xetabase from target branch"
+        run: |
+          chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
+          echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}"
+          echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}"
+          echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}"
+          xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }})
+          echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
+          echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT
+        env:
+          ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
+
   test:
     name: "Run all tests before merging"
-    uses: ./.github/workflows/test-analysis.yml
-    needs: build
+    needs: calculate-xetabase-branch
+    uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop
+    with:
+      branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }}
+      task: ${{ github.event.pull_request.head.ref }}
     secrets: inherit
diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e
+set -x
+# Function to calculate the corresponding branch of Xetabase project
+get_xetabase_branch() {
+  # Input parameter (branch name)
+  input_branch="$1"
+
+  # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it
+  if [[ $input_branch == TASK* ]]; then
+    if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then
+      echo $input_branch;
+      return 0;
+    fi
+  fi
+
+  # Check if the branch name is "develop" in that case return the same branch name
+  if [[ "$input_branch" == "develop" ]]; then
+    echo "develop"
+    return 0
+  fi
+
+  # Check if the branch name starts with "release-" and follows the patterns "release-a.b.x" or "release-a.b.c.x"
+  if [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.([0-9]+)\.x$ ]]; then
+    # Extract the MAJOR part of the branch name
+    MAJOR=${BASH_REMATCH[1]}
+    # Calculate the XETABASE_MAJOR by subtracting 3 from MAJOR
+    XETABASE_MAJOR=$((MAJOR - 3))
+    # Check if the XETABASE_MAJOR is negative
+    if (( XETABASE_MAJOR < 0 )); then
+      echo "Error: 'MAJOR' digit after subtraction results in a negative number."
+      return 1
+    fi
+    # Construct and echo the new branch name
+    echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}"
+    return 0
+  fi
+
+  # If the branch name does not match any of the expected patterns
+  echo "Error: The branch name is not correct."
+  return 1
+}
+
+
+# Call the function with the input branch name
+get_xetabase_branch "$1"
diff --git a/.github/workflows/test-pull-request-approved.yml b/.github/workflows/test-pull-request-approved.yml
@@ -0,0 +1,48 @@
+name: Pull request approve workflow
+run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}'
+
+on:
+  push:
+    branches:
+      - TASK-*
+
+
+jobs:
+  calculate-xetabase-branch:
+    name: Calculate Xetabase branch
+    runs-on: ubuntu-22.04
+    outputs:
+      xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }}
+    steps:
+      - name: Clone java-common-libs
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found"
+          persist-credentials: false
+      - id: get_xetabase_branch
+        name: "Get current branch for Xetabase from target branch"
+        run: |
+          if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/opencb/opencga.git "TASK-6807" )" ] ; then
+           echo "OPENCGA TASK-6807 branch";
+          fi
+          
+          if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then
+            echo "Here it is TASK-6807 branch";
+          fi
+          chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
+          echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" | tee -a ${GITHUB_STEP_SUMMARY}
+          xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6807" )
+          echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
+          echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT
+        env:
+          ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
+
+  test:
+    name: "Run all tests before merging"
+    needs: calculate-xetabase-branch
+    uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop
+    with:
+      branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }}
+      task: TASK-6807
+    secrets: inherit
diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml
@@ -19,33 +19,49 @@ on:
         type: string
         description: 'Branch of opencga-enterprise to be tested and built.'
         required: true
-env:
-  AZCOPY_SPA_CLIENT_SECRET: ${{ secrets.AZCOPY_SPA_CLIENT_SECRET }}
-  AZCOPY_AUTO_LOGIN_TYPE: "SPN"
-  AZCOPY_SPA_APPLICATION_ID: ${{ secrets.AZCOPY_SPA_APPLICATION_ID }}
-  AZCOPY_TENANT_ID: ${{ secrets.AZCOPY_TENANT_ID }}
 
 jobs:
   test:
     name: Execute JUnit and Jacoco tests
     runs-on: ubuntu-22.04
     steps:
-      - name: Clone OpenCGA Enterprise branch '${{ github.event.inputs.branch }}'
+      - name: Retrieve secrets from Keeper
+        id: ksecrets
+        uses: Keeper-Security/ksm-action@master
+        with:
+          keeper-secret-config: ${{ secrets.KEEPER_SM_GH_OPENCB }}
+          secrets: |
+            AZURE_KUBE_CONFIG/field/Secret Value > env:AZURE_KUBE_CONFIG
+            DOCKER_HUB_USER/field/Secret Value > env:DOCKER_HUB_USER
+            DOCKER_HUB_PASSWORD/field/Secret Value > env:DOCKER_HUB_PASSWORD
+            SSH_TESTING_SERVER_HOST/field/Secret Value > env:SSH_HOST
+            SSH_TESTING_SERVER_PORT/field/Secret Value > env:SSH_PORT
+            SSH_TESTING_SERVER_USER/field/Secret Value > env:SSH_USER
+            SSH_TESTING_SERVER_PASSWORD/field/Secret Value > env:SSH_PASS
+      - name: Log inputs
+        run: |
+          echo "__OpenCGA-enterprise branch:__ \"${{ inputs.branch }}\"" | tee -a $GITHUB_STEP_SUMMARY
+          echo "__Task to test:__ \"${{ inputs.task }}\"" | tee -a $GITHUB_STEP_SUMMARY
+      - name: Clone OpenCGA Enterprise branch '${{ inputs.branch }}'
         uses: actions/checkout@v4
         with:
           repository: zetta-genomics/opencga-enterprise
-          ref: ${{ github.event.inputs.branch }}
+          ref: ${{ inputs.branch }}
           token: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
           path: opencga-enterprise
           fetch-depth: "10"
       - id: get_opencga_branch
         name: Get OpenCGA branch from 'pom.xml' property
         run: |
           pwd
-          chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh
-          opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh ${{ github.event.inputs.task }})
+          ls -lrtha
+          ls -lrtha ./opencga-enterprise
+          chmod +x ./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh
+          opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh)
+          echo "opencga_branch=${opencga_branch}"
           echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v4
+      - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}'
+        uses: actions/checkout@v4
         with:
           repository: opencb/opencga
           ref: ${{ steps.get_opencga_branch.outputs.opencga_branch }}
@@ -70,39 +86,39 @@ jobs:
           chmod +x ./kubectl
           echo "${{ secrets.AZURE_KUBE_CONFIG }}" > admin.conf
           ./kubectl -n cellbase-db port-forward services/cellbase-rs0-svc 27018:27017 --kubeconfig ./admin.conf &
-      - name: Install Azure AZCOPY
-        uses: kheiakiyama/install-azcopy-action@v1
-        with:
-          version: 'v10'
       - name: DockerHub login
         uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ env.DOCKER_HUB_PASSWORD }}
+      - name: Install sshpass
+        run: sudo apt-get install sshpass
+      - name: Add SSH Host to known_hosts
+        run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan -p ${{ env.SSH_PORT }} ${{ env.SSH_HOST }} >> ~/.ssh/known_hosts
+        env:
+          SSH_HOST: ${{ env.SSH_HOST }}
+          SSH_PORT: ${{ env.SSH_PORT }}
       - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise
         run: |
-          ln -s opencga opencga-enterprise/opencga-home
           cd opencga-enterprise
-          ./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ github.event.inputs.task }} -c localhost:27018 -H hdp3.1
+          ln -s ../opencga opencga-home
+          ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1
       - name: Upload reports results to Github
         uses: actions/upload-artifact@v4
         with:
           name: report-test
-          path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/reports/test
+          path: ./opencga-enterprise/reports/test
       - name: Upload log
         uses: actions/upload-artifact@v4
         with:
           name: build-log
-          path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/build.log
-      - name: Upload junit reports to a remote scp server
-        uses: garygrossgarten/github-action-scp@release
-        with:
-          local: opencga-enterprise/reports/test
-          remote: /var/www/html/reports/xetabase/${{ github.event.inputs.task }}/
-          host: ${{ secrets.SSH_TESTING_SERVER_HOST}}
-          port: ${{ secrets.SSH_TESTING_SERVER_PORT}}
-          username: ${{ secrets.SSH_TESTING_SERVER_USER }}
-          password: ${{ secrets.SSH_TESTING_SERVER_PASSWORD }}
-          concurrency: 2
+          path: ./opencga-enterprise/build.log
+      - name: Log summary
+        run: |
+          cat ./opencga-enterprise/build.log | tee -a $GITHUB_STEP_SUMMARY
+          
+          
 
 
diff --git a/commons-datastore/commons-datastore-core/pom.xml b/commons-datastore/commons-datastore-core/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>5.2.0</version>
+        <version>5.2.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-datastore/commons-datastore-mongodb/pom.xml b/commons-datastore/commons-datastore-mongodb/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>5.2.0</version>
+        <version>5.2.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-datastore/commons-datastore-solr/pom.xml b/commons-datastore/commons-datastore-solr/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>5.2.0</version>
+        <version>5.2.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-datastore/pom.xml b/commons-datastore/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons</artifactId>
-        <version>5.2.0</version>
+        <version>5.2.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-lib/pom.xml b/commons-lib/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons</artifactId>
-        <version>5.2.0</version>
+        <version>5.2.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -69,5 +69,11 @@
             <artifactId>jackson-databind</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.8.0</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 </project>
diff --git a/commons-lib/src/main/java/org/opencb/commons/utils/FileUtils.java b/commons-lib/src/main/java/org/opencb/commons/utils/FileUtils.java
@@ -18,6 +18,7 @@
 
 import org.apache.commons.lang3.StringUtils;
 import org.opencb.commons.exec.Command;
+import org.slf4j.LoggerFactory;
 
 import java.io.*;
 import java.nio.charset.Charset;
@@ -188,6 +189,27 @@ public static String[] getUserAndGroup(Path path, boolean numericId) throws IOEx
         return new String[]{split[2], split[3]};
     }
 
+
+    public static void copyFile(File src, File dest) throws IOException {
+        try {
+            org.apache.commons.io.FileUtils.copyFile(src, dest);
+        } catch (IOException e) {
+            try {
+                if (src.length() == dest.length()) {
+                    LoggerFactory.getLogger(FileUtils.class).warn(e.getMessage());
+                    return;
+                }
+                throw e;
+            } catch (Exception e1) {
+                throw e;
+            }
+        }
+    }
+
+    //-------------------------------------------------------------------------
+    // P R I V A T E     M E T H O D S
+    //-------------------------------------------------------------------------
+
     private static String getLsOutput(Path path, boolean numericId) throws IOException {
         FileUtils.checkPath(path);
 

diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>org.opencb.commons</groupId>
     <artifactId>commons</artifactId>
-    <version>5.2.0</version>
+    <version>5.2.1</version>
     <packaging>pom</packaging>
 
     <name>OpenCB commons project</name>