From b649b68e94de87a4d69aeb3db943283bb9a4603a Mon Sep 17 00:00:00 2001 From: aneeshafedo Date: Fri, 18 Aug 2023 14:24:00 +0530 Subject: [PATCH 1/7] Add connector test workflow --- .github/workflows/test-connectors.yaml | 53 ++++++++++++++++++++++++++ build.gradle | 42 +++++++++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-connectors.yaml diff --git a/.github/workflows/test-connectors.yaml b/.github/workflows/test-connectors.yaml new file mode 100644 index 000000000..92a06f2ae --- /dev/null +++ b/.github/workflows/test-connectors.yaml @@ -0,0 +1,53 @@ +name: Regenerate Connectors + +on: + # Run nightly at 2:00 AM + schedule: + - cron: '0 2 * * *' + # Trigger manually with specified Ballerina Docker version + workflow_dispatch: + inputs: + bal_docker_version: + description: Ballerina Docker version + required: true + default: 'nightly' + client_method: + description: Expected Client Method Type + required: true + default: 'remote' + options: ['remote', 'resource'] + +jobs: + test_openapi_tool_nightly: + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + container: + image: ballerina/ballerina:nightly + options: --user root + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build with Gradle + env: + JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true + GRADLE_USER_HOME: ~/.gradle + run: | + ./gradlew build -PtestTool=true -PclientMethod=${{ github.event.inputs.client_method }} + + test_openapi_tool_with_specified_version: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + container: + image: ballerina/ballerina:${{ github.event.inputs.bal_docker_version }} + options: --user root + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build with Gradle + env: + JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true + GRADLE_USER_HOME: ~/.gradle + run: | + ./gradlew build -PtestTool=true -PclientMethod=${{ github.event.inputs.client_method }} diff --git a/build.gradle b/build.gradle index 18a90d51b..feae20c6d 100644 --- a/build.gradle +++ b/build.gradle @@ -24,6 +24,9 @@ boolean release = false; boolean remote = false; boolean buildAll = false; boolean testTool = false; +boolean testConnectorGeneration = false; +String clientMethod = "remote" + if (project.hasProperty("remote")){ remote = new Boolean(project.property("remote").toString()) } @@ -36,6 +39,12 @@ if (project.hasProperty("buildAll")){ if (project.hasProperty("testTool")){ testTool = new Boolean(project.property("testTool").toString()) } +if (project.hasProperty("testConnectorGeneration")){ + testConnectorGeneration = new Boolean(project.property("testConnectorGeneration").toString()) +} +if (project.hasProperty("clientMethod")){ + clientMethod = new Boolean(project.property("clientMethod").toString()) +} String toolTestsDirPath = project.projectDir.absolutePath + "/tool-tests"; List toolTestPackages = new ArrayList<>(); @@ -50,7 +59,7 @@ if (buildAll){ } task codeBuild { - if (!testTool) { + if (!testTool && !testConnectorGeneration) { println "Task: Building connectors..." for (String path : updatedBallerinaPackages) { Utils.executePrechecks(path); @@ -151,3 +160,34 @@ task testOpenAPITool { } } } + +task generateNbuildLatestOpenAPIConnectors { + if (testConnectorGeneration) { + println "Task: Re-generate and build connectors..." + for (String path : ballerinaPackages) { + println "----- Conenector : " + path + try { + exec { + def clientCmd = (clientMethod == "remote") ? "--client-methods remote" : "" + commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal openapi -i openapi.* --mode client ${clientCmd}" + workingDir(path) + } + println "Connector generation is success for " + path + + } catch (Exception ex) { + println "Failed to re-generate the connector [" + path + "]. Error : " + ex.toString(); + continue + } + + try { + exec { + commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal pack ${path}" + } + println "bal pack success for " + path + } catch (Exception ex) { + println "Failed to build the re-generate the connector [" + path + "]. Error : " + ex.toString(); + + } + } + } +} \ No newline at end of file From 2926f9f2bbf24bcbd53b1ef749727325b7111384 Mon Sep 17 00:00:00 2001 From: aneeshafedo Date: Fri, 18 Aug 2023 14:38:22 +0530 Subject: [PATCH 2/7] Update workflow --- .github/workflows/test-connectors.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-connectors.yaml b/.github/workflows/test-connectors.yaml index 92a06f2ae..f0be4cd98 100644 --- a/.github/workflows/test-connectors.yaml +++ b/.github/workflows/test-connectors.yaml @@ -18,7 +18,7 @@ on: options: ['remote', 'resource'] jobs: - test_openapi_tool_nightly: + regenerate_connectors_with_nightly: if: github.event_name == 'schedule' runs-on: ubuntu-latest container: @@ -33,9 +33,9 @@ jobs: JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true GRADLE_USER_HOME: ~/.gradle run: | - ./gradlew build -PtestTool=true -PclientMethod=${{ github.event.inputs.client_method }} + ./gradlew build -PtestConnectorGeneration=true -PclientMethod=${{ github.event.inputs.client_method }} - test_openapi_tool_with_specified_version: + regenerate_connectors_with_specified_version: if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest container: @@ -50,4 +50,4 @@ jobs: JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true GRADLE_USER_HOME: ~/.gradle run: | - ./gradlew build -PtestTool=true -PclientMethod=${{ github.event.inputs.client_method }} + ./gradlew build -PtestConnectorGeneration=true -PclientMethod=${{ github.event.inputs.client_method }} From 6afed66d9bd67e114e721503e9870e40bd712e9b Mon Sep 17 00:00:00 2001 From: aneeshafedo Date: Fri, 18 Aug 2023 15:16:47 +0530 Subject: [PATCH 3/7] Add summary --- .github/workflows/test-connectors.yaml | 36 ++++++++--------- build.gradle | 56 +++++++++++++++++++------- 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test-connectors.yaml b/.github/workflows/test-connectors.yaml index f0be4cd98..086f9e06a 100644 --- a/.github/workflows/test-connectors.yaml +++ b/.github/workflows/test-connectors.yaml @@ -1,9 +1,9 @@ name: Regenerate Connectors on: - # Run nightly at 2:00 AM - schedule: - - cron: '0 2 * * *' + # # Run nightly at 2:00 AM + # schedule: + # - cron: '0 2 * * *' # Trigger manually with specified Ballerina Docker version workflow_dispatch: inputs: @@ -18,22 +18,22 @@ on: options: ['remote', 'resource'] jobs: - regenerate_connectors_with_nightly: - if: github.event_name == 'schedule' - runs-on: ubuntu-latest - container: - image: ballerina/ballerina:nightly - options: --user root - steps: - - name: Checkout code - uses: actions/checkout@v2 + # regenerate_connectors_with_nightly: + # if: github.event_name == 'schedule' + # runs-on: ubuntu-latest + # container: + # image: ballerina/ballerina:nightly + # options: --user root + # steps: + # - name: Checkout code + # uses: actions/checkout@v2 - - name: Build with Gradle - env: - JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true - GRADLE_USER_HOME: ~/.gradle - run: | - ./gradlew build -PtestConnectorGeneration=true -PclientMethod=${{ github.event.inputs.client_method }} + # - name: Build with Gradle + # env: + # JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true + # GRADLE_USER_HOME: ~/.gradle + # run: | + # ./gradlew build -PtestConnectorGeneration=true -PclientMethod=${{ github.event.inputs.client_method }} regenerate_connectors_with_specified_version: if: github.event_name == 'workflow_dispatch' diff --git a/build.gradle b/build.gradle index feae20c6d..00a981e12 100644 --- a/build.gradle +++ b/build.gradle @@ -164,30 +164,58 @@ task testOpenAPITool { task generateNbuildLatestOpenAPIConnectors { if (testConnectorGeneration) { println "Task: Re-generate and build connectors..." + def generationFailedConnectors = [] // List to store generation failed connectors + def compilationFailedConnectors = [] // List to store compilation failed connectors + + for (String path : ballerinaPackages) { - println "----- Conenector : " + path + println "----- Connector : " + path try { exec { - def clientCmd = (clientMethod == "remote") ? "--client-methods remote" : "" - commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal openapi -i openapi.* --mode client ${clientCmd}" + commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal openapi -i openapi.* --mode client" workingDir(path) } - println "Connector generation is success for " + path - - } catch (Exception ex) { - println "Failed to re-generate the connector [" + path + "]. Error : " + ex.toString(); - continue + println "Connector generation is successful for " + path + } catch (Exception genEx) { + println "Failed to re-generate the connector [" + path + "]. Error : " + genEx.toString() + generationFailedConnectors << path // Add the failed connector to the list } try { exec { commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal pack ${path}" - } - println "bal pack success for " + path - } catch (Exception ex) { - println "Failed to build the re-generate the connector [" + path + "]. Error : " + ex.toString(); - + } + println "bal pack success for " + path + } catch (Exception buildEx) { + println "Failed to build the re-generate the connector [" + path + "]. Error : " + buildEx.toString() + compilationFailedConnectors << path // Add the failed connector to the list + } + } + + // Print summary of re-generation failed connectors + if (generationFailedConnectors.size() > 0) { + println "Summary: ${generationFailedConnectors.size()} connectors failed re-generation:" + generationFailedConnectors.each { connector -> + println "- $connector" } + } + + // Print summary of compilation failed connectors + if (compilationFailedConnectors.size() > 0) { + println "Summary: ${compilationFailedConnectors.size()} connectors failed compilation:" + compilationFailedConnectors.each { connector -> + println "- $connector" + } + } + + if (generationFailedConnectors.size() > 0 || compilationFailedConnectors.size() > 0) { + // Throw an exception to fail the build + throw new GradleException("${failedConnectors.size()} connectors failed regeneration & + ${failedConnectors.size()} connectors failed compilation. See the summary above.") + } + + if (generationFailedConnectors.size() == 0 && compilationFailedConnectors.size() == 0) { + println "All connectors executed successfully." } } -} \ No newline at end of file +} From 83b649954c8de08c0ac75c02edd9bca9d5a5120d Mon Sep 17 00:00:00 2001 From: aneeshafedo Date: Fri, 18 Aug 2023 16:46:00 +0530 Subject: [PATCH 4/7] support bot type of client methods --- build.gradle | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index 00a981e12..c340e01b3 100644 --- a/build.gradle +++ b/build.gradle @@ -25,7 +25,7 @@ boolean remote = false; boolean buildAll = false; boolean testTool = false; boolean testConnectorGeneration = false; -String clientMethod = "remote" +String clientMethod = "remote"; if (project.hasProperty("remote")){ remote = new Boolean(project.property("remote").toString()) @@ -43,7 +43,7 @@ if (project.hasProperty("testConnectorGeneration")){ testConnectorGeneration = new Boolean(project.property("testConnectorGeneration").toString()) } if (project.hasProperty("clientMethod")){ - clientMethod = new Boolean(project.property("clientMethod").toString()) + clientMethod = new String(project.property("clientMethod").toString()) } String toolTestsDirPath = project.projectDir.absolutePath + "/tool-tests"; @@ -167,18 +167,18 @@ task generateNbuildLatestOpenAPIConnectors { def generationFailedConnectors = [] // List to store generation failed connectors def compilationFailedConnectors = [] // List to store compilation failed connectors - for (String path : ballerinaPackages) { println "----- Connector : " + path try { exec { - commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal openapi -i openapi.* --mode client" + commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal openapi -i openapi.* --mode client --client-methods ${clientMethod}" workingDir(path) } println "Connector generation is successful for " + path } catch (Exception genEx) { println "Failed to re-generate the connector [" + path + "]. Error : " + genEx.toString() generationFailedConnectors << path // Add the failed connector to the list + continue } try { @@ -210,8 +210,9 @@ task generateNbuildLatestOpenAPIConnectors { if (generationFailedConnectors.size() > 0 || compilationFailedConnectors.size() > 0) { // Throw an exception to fail the build - throw new GradleException("${failedConnectors.size()} connectors failed regeneration & - ${failedConnectors.size()} connectors failed compilation. See the summary above.") + throw new GradleException("${failedConnectors.size()} connectors failed regeneration & " + + "${failedConnectors.size()} connectors failed compilation. See the summary above.") + } if (generationFailedConnectors.size() == 0 && compilationFailedConnectors.size() == 0) { From 8b8ea235b749a63dd94a45dba86f3af22013b93b Mon Sep 17 00:00:00 2001 From: aneeshafedo Date: Fri, 18 Aug 2023 18:42:34 +0530 Subject: [PATCH 5/7] Update workflow --- build.gradle | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index c340e01b3..6b67cf4cb 100644 --- a/build.gradle +++ b/build.gradle @@ -210,8 +210,8 @@ task generateNbuildLatestOpenAPIConnectors { if (generationFailedConnectors.size() > 0 || compilationFailedConnectors.size() > 0) { // Throw an exception to fail the build - throw new GradleException("${failedConnectors.size()} connectors failed regeneration & " + - "${failedConnectors.size()} connectors failed compilation. See the summary above.") + throw new GradleException("${generationFailedConnectors.size()} connectors failed regeneration & " + + "${compilationFailedConnectors.size()} connectors failed compilation. See the summary above.") } @@ -220,3 +220,9 @@ task generateNbuildLatestOpenAPIConnectors { } } } + +gradle.buildFinished { + if (gradle.buildFinished.buildResult.failure) { + throw new GradleException("Workflow failed due to build failure.") + } +} \ No newline at end of file From 6d8468771ac6ee7a03f6d2214f4bffae2585c8e7 Mon Sep 17 00:00:00 2001 From: aneeshafedo Date: Fri, 18 Aug 2023 18:42:34 +0530 Subject: [PATCH 6/7] Update workflow --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index c340e01b3..d89e84dc0 100644 --- a/build.gradle +++ b/build.gradle @@ -210,8 +210,8 @@ task generateNbuildLatestOpenAPIConnectors { if (generationFailedConnectors.size() > 0 || compilationFailedConnectors.size() > 0) { // Throw an exception to fail the build - throw new GradleException("${failedConnectors.size()} connectors failed regeneration & " + - "${failedConnectors.size()} connectors failed compilation. See the summary above.") + throw new GradleException("${generationFailedConnectors.size()} connectors failed regeneration & " + + "${compilationFailedConnectors.size()} connectors failed compilation. See the summary above.") } From becb7696fece8409e05a95e822e3220baab2d218 Mon Sep 17 00:00:00 2001 From: aneeshafedo Date: Fri, 18 Aug 2023 18:51:46 +0530 Subject: [PATCH 7/7] update build failure --- build.gradle | 6 ------ 1 file changed, 6 deletions(-) diff --git a/build.gradle b/build.gradle index 6b67cf4cb..d89e84dc0 100644 --- a/build.gradle +++ b/build.gradle @@ -220,9 +220,3 @@ task generateNbuildLatestOpenAPIConnectors { } } } - -gradle.buildFinished { - if (gradle.buildFinished.buildResult.failure) { - throw new GradleException("Workflow failed due to build failure.") - } -} \ No newline at end of file