Merge pull request #812 from aneeshafedo/add-workflow

Add workflow to regenerate connectors

.github/workflows/test-connectors.yml

@@ -0,0 +1,58 @@
+name: Regenerate Connectors
+
+on:
+  workflow_dispatch:
+    inputs:
+      bal_docker_version:
+        description: Ballerina Docker version
+        required: true
+        default: 'nightly'
+      artifact_id:
+        description: Artifact Id of the custom Ballerina distribution
+        required: false
+        default: ''
+      client_method:
+        description: Expected Client Method Type
+        required: true
+        default: 'remote'
+        options: ['remote', 'resource']
+      connector_list:
+        description: Comma separated list of connectors to regenerate
+        required: false
+        default: ''
+
+jobs:
+  regenerate_connectors_with_specified_version:
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    container:
+      image: ballerina/ballerina:${{ github.event.inputs.bal_docker_version }}
+      options: --user root
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Build with Gradle
+        env:
+          JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true
+          GRADLE_USER_HOME: ~/.gradle
+        run: |
+          if [[ ${{ github.event.inputs.artifact_id }} != '' ]]; then
+            wget --header="Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -O ballerina.zip "https://api.github.com/repos/ballerina-platform/ballerina-distribution/actions/artifacts/${{ github.event.inputs.artifact_id }}/zip"
+            unzip -o ballerina.zip
+            balDistZip=$(find . -name 'ballerina-*.zip' | head -n 1)
+            echo "Unzipping ${balDistZip}"
+            unzip -o "${balDistZip}"
+            EXTRACTED_DISTRIBUTION_PATH="${balDistZip%.zip}"
+            ABSOLUTE_EXTRACTED_DISTRIBUTION_PATH=$(realpath "$EXTRACTED_DISTRIBUTION_PATH")
+            echo "Start generation flow..."
+            ./gradlew build -PtestConnectorGeneration=true -PclientMethod=${{ github.event.inputs.client_method }} -PcustomDistributionPath=$ABSOLUTE_EXTRACTED_DISTRIBUTION_PATH -PconnectorList=${{ github.event.inputs.connector_list }}
+          else
+            echo "Start generation flow..."
+            ./gradlew build -PtestConnectorGeneration=true -PclientMethod=${{ github.event.inputs.client_method }} -PconnectorList=${{ github.event.inputs.connector_list }}
+          fi
+          
+      - name: Upload Summary Report
+        uses: actions/upload-artifact@v2
+        with:
+          name: summary-report
+          path: .github/summary.log

build.gradle

@@ -24,6 +24,11 @@ boolean release = false;
 boolean remote = false;
 boolean buildAll = false;
 boolean testTool = false;
+boolean testConnectorGeneration = false;
+String clientMethod = "remote";
+String customDistributionPath = null;
+String connectorList = null;
+
 if (project.hasProperty("remote")){
     remote = new Boolean(project.property("remote").toString())
 }
@@ -36,6 +41,18 @@ if (project.hasProperty("buildAll")){
 if (project.hasProperty("testTool")){
     testTool = new Boolean(project.property("testTool").toString())
 }
+if (project.hasProperty("testConnectorGeneration")){
+    testConnectorGeneration = new Boolean(project.property("testConnectorGeneration").toString())
+}
+if (project.hasProperty("clientMethod")){
+    clientMethod = new String(project.property("clientMethod").toString())
+}
+if (project.hasProperty("customDistributionPath")){
+    customDistributionPath = new String(project.property("customDistributionPath").toString())
+}
+if (project.hasProperty("connectorList")){
+    connectorList = new String(project.property("connectorList").toString())
+}
 
 String toolTestsDirPath = project.projectDir.absolutePath + "/tool-tests";
 List<String> toolTestPackages = new ArrayList<>();
@@ -50,7 +67,7 @@ if (buildAll){
 }
 
 task codeBuild {
-    if (!testTool) {
+    if (!testTool && !testConnectorGeneration) {
         println "Task: Building connectors..."
         for (String path : updatedBallerinaPackages) {
             Utils.executePrechecks(path);
@@ -151,3 +168,119 @@ task testOpenAPITool {
         }
     }
 }
+
+task generateAndBuildLatestOpenAPIConnectors {
+    if (testConnectorGeneration) {
+        println "Task: Re-generate and build connectors..." 
+
+        def generationFailedConnectors = []  // List to store generation failed connectors
+        def compilationFailedConnectors = []  // List to store compilation failed connectors
+        def execConnectors = 0;
+
+        if (customDistributionPath == null || customDistributionPath == "") {
+            println "Custom distribution path is not provided. Using the default distribution path."
+        } else {
+            println "Using the custom distribution path : " + customDistributionPath
+            ballerinaDistributionPath = customDistributionPath;
+        }
+
+        if (connectorList != null && connectorList != "") {
+            println "Connector list is provided. Using the provided connector list."
+            def selectedConnectorPath = []
+            for (String connector : connectorList.split(",")) {
+                def path = openApiPackageDirPath + "/" + connector.trim();
+                if (ballerinaPackages.contains(path)) {
+                    selectedConnectorPath.add(path)
+                } else {
+                    println "Connector [" + connector + "] is not found in the repository."
+                }
+            }
+            ballerinaPackages = selectedConnectorPath;
+        }
+
+        for (String path : ballerinaPackages) {
+            def connector = path;
+
+            // Extract the connector name from the path
+            def regex = /\/openapi\/(.*?)$/
+            def matcher = (path =~ regex)
+            if (matcher.find()) {
+                connector = matcher.group(1)
+            }
+
+            // Re-generate the connector
+            println "--- Generating the connector : " + connector
+            try {
+                exec {
+                    execConnectors++;
+                    workingDir(path)
+                    commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal openapi -i openapi.* --mode client --client-methods ${clientMethod}"     
+                }
+                println "Connector generation is successful for " + connector
+            } catch (Exception genEx) {
+                println "Failed to re-generate the connector [" + connector + "]. Error : " + genEx.toString()
+                generationFailedConnectors << connector  // Add the failed connector to the list
+                continue
+            }
+
+            // Build the connector
+            try {
+                exec {
+                    commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal pack ${path}"
+                }
+                println "bal pack success for " + connector
+            } catch (Exception buildEx) {
+                println "Failed to build the re-generate the connector [" + connector + "]. Error : " + buildEx.toString()
+                compilationFailedConnectors << connector  // Add the failed connector to the list
+            }
+        }
+
+        // Generate the summary report
+        def summary = constructSummaryReport(ballerinaDistributionPath, ballerinaPackages.size(), generationFailedConnectors, compilationFailedConnectors, execConnectors, clientMethod)
+        def reportFile = file("${project.projectDir}/.github/summary.log")
+        reportFile.text = summary
+
+        if (generationFailedConnectors.size() == 0 && compilationFailedConnectors.size() == 0) {
+            println "All connectors executed successfully."
+        }
+    }
+}
+
+def constructSummaryReport(ballerinaDistributionPath, totalConnectors, generationFailedConnectors, compilationFailedConnectors, execConnectors, clientMethod) {
+    def summary = "# Summary \n\n"
+    summary += "Timestamp: ${new Date()}\n"
+    summary += "Ballerina Version: ${getBallerinaVersion(ballerinaDistributionPath).toString()}\n"
+    summary += "Client Method Type: ${clientMethod}\n"
+    summary += "Total Connectors in the Repository: ${totalConnectors}\n"
+    summary += "Number of Connectors Executed: ${execConnectors}\n\n"
+
+    if (generationFailedConnectors.size() > 0 || compilationFailedConnectors.size() > 0) {
+        summary += "Failed Connectors:\n\n"
+        if (generationFailedConnectors.size() > 0) {
+            summary += "Generation Failed Connectors (count: ${generationFailedConnectors.size()}) :\n"
+            generationFailedConnectors.each { connector ->
+                summary += "- ${connector}\n"
+            }
+        }
+        if (compilationFailedConnectors.size() > 0) {
+            summary += "Compilation Failed Connectors (count: ${compilationFailedConnectors.size()}) :\n"
+            compilationFailedConnectors.each { connector ->
+                summary += "- ${connector}\n"
+            }
+        }
+    } else {
+        summary += "All connectors are regenerated and compiled successfully.\n"
+    }
+
+    return summary
+}
+
+def getBallerinaVersion(ballerinaDistributionPath) {
+    def command = ballerinaDistributionPath + "/bin/bal -v"
+    def process = command.execute()
+    process.waitFor()
+    def output = process.text.trim()
+    def ballerinaVersion = output.split('\n').find { it.startsWith("Ballerina") }
+    return ballerinaVersion
+}
+