From a225d334a72a52a21dadba7e50388abf86107a56 Mon Sep 17 00:00:00 2001 From: manjudr Date: Thu, 23 Dec 2021 12:51:09 +0530 Subject: [PATCH 1/8] SB-27924:feat: Refactor of CircleCI --- .circleci/config.yml | 135 ++++++++++++++++++++++++++++++++----------- 1 file changed, 102 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 22d4bc64d..86c34162c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,10 @@ version: 2.1 -jobs: - sunbird-ed-dp-build: - machine: - image: ubuntu-2004:202008-01 + +commands: + clone-dependency-repos: steps: - - checkout - run: - name: clone analytics core + name: Clone All Dependency Repo's command: | if [ -z $CIRCLE_PR_NUMBER ]; then target_branch=$CIRCLE_BRANCH @@ -35,37 +33,108 @@ jobs: else cd sunbird-core-dataproducts && git checkout $target_branch fi - - restore_cache: - keys: - - dp-dependency-cache-{{ checksum "data-products/pom.xml" }} - - run: + build-framework-artifacts: + steps: + - run: name: Build analytics-core dependency jar command: | export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 export PATH=$JAVA_HOME/bin:$PATH cd sunbird-analytics-core && mvn install -DskipTests + build-core-jobs-artifacts: + steps: - run: - name: Build core-data-products dependency jar - command: | - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 - export PATH=$JAVA_HOME/bin:$PATH - cd sunbird-core-dataproducts && mvn install -DskipTests - - run: - name: Ed-data-products-build - command: | - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 - export PATH=$JAVA_HOME/bin:$PATH - cd data-products && mvn scoverage:report - - save_cache: - key: dp-dependency-cache-{{ checksum "data-products/pom.xml" }} - paths: ~/.m2 - - run: - name: sonar + name: Build analytics-core-dataproducts dependency jar command: | - export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64 - cd data-products && mvn -X sonar:sonar -Dsonar.projectKey=Sunbird-Ed_sunbird-data-products -Dsonar.host.url=https://sonarcloud.io -Dsonar.exclusions=/home/circleci/project/adhoc-scripts/**,/home/circleci/project/replay-scripts/** -Dsonar.scala.coverage.reportPaths=/home/circleci/project/data-products/target/scoverage.xml -Dsonar.organization=sunbird-ed -workflows: - version: 2.1 - workflow: - jobs: - - sunbird-ed-dp-build + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 + export PATH=$JAVA_HOME/bin:$PATH + cd sunbird-core-dataproducts && mvn install -DskipTests + run-tests: + steps: + - run: + name: Build analytics-core-dataproducts dependency jar + command: | + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 + export PATH=$JAVA_HOME/bin:$PATH + cd data-products && mvn scoverage:report +#jobs: +# sunbird-ed-dp-build: +# machine: +# image: ubuntu-2004:202008-01 +# steps: +# - checkout +# - run: +# name: clone analytics core +# command: | +# if [ -z $CIRCLE_PR_NUMBER ]; then +# target_branch=$CIRCLE_BRANCH +# else +# prdata=$(curl -X GET -u $GITHUB_USER_TOKEN:x-oauth-basic https://api.github.com/repos/Sunbird-Ed/sunbird-data-products/pulls/$CIRCLE_PR_NUMBER) +# target_branch=$(echo "${prdata}" | jq -r '.base.ref') +# fi +# echo $target_branch +# git clone https://github.com/project-sunbird/sunbird-analytics-core.git +# branchExists=$(cd sunbird-analytics-core && git ls-remote --heads origin $target_branch) +# echo $branchExists +# if [ -z ${branchExists} ]; then +# latest_branch=$(cd sunbird-analytics-core && git for-each-ref --count=1 --sort=-committerdate 'refs/remotes/**/release*' --format='%(refname:short)' | head -1) +# echo $latest_branch +# cd sunbird-analytics-core && git checkout $latest_branch +# else +# cd sunbird-analytics-core && git checkout $target_branch +# fi +# cd ../ && git clone https://github.com/project-sunbird/sunbird-core-dataproducts.git +# branchExists=$(cd sunbird-core-dataproducts && git ls-remote --heads origin $target_branch) +# echo $branchExists +# if [ -z ${branchExists} ]; then +# latest_branch=$(cd sunbird-core-dataproducts && git for-each-ref --count=1 --sort=-committerdate 'refs/remotes/**/release*' --format='%(refname:short)' | head -1) +# echo $latest_branch +# cd sunbird-core-dataproducts && git checkout $latest_branch +# else +# cd sunbird-core-dataproducts && git checkout $target_branch +# fi +# - restore_cache: +# keys: +# - dp-dependency-cache-{{ checksum "data-products/pom.xml" }} +# - run: +# name: Build analytics-core dependency jar +# command: | +# export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 +# export PATH=$JAVA_HOME/bin:$PATH +# cd sunbird-analytics-core && mvn install -DskipTests +# - run: +# name: Build core-data-products dependency jar +# command: | +# export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 +# export PATH=$JAVA_HOME/bin:$PATH +# cd sunbird-core-dataproducts && mvn install -DskipTests +# - run: +# name: Ed-data-products-build +# command: | +# export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 +# export PATH=$JAVA_HOME/bin:$PATH +# cd data-products && mvn scoverage:report +# - save_cache: +# key: dp-dependency-cache-{{ checksum "data-products/pom.xml" }} +# paths: ~/.m2 +# - run: +# name: sonar +# command: | +# export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64 +# cd data-products && mvn -X sonar:sonar -Dsonar.projectKey=Sunbird-Ed_sunbird-data-products -Dsonar.host.url=https://sonarcloud.io -Dsonar.exclusions=/home/circleci/project/adhoc-scripts/**,/home/circleci/project/replay-scripts/** -Dsonar.scala.coverage.reportPaths=/home/circleci/project/data-products/target/scoverage.xml -Dsonar.organization=sunbird-ed +# +jobs: + build-artifacts: + machine: true + steps: + - clone-dependency-repos + - build-framework-artifacts + - build-core-jobs-artifacts + run-tests: + docker: + - image: "circleci/openjdk:11-jdk" + steps: + - checkout + - run-tests + + From 4beb1d631a50f0daf110ec3e36529eba00bb3670 Mon Sep 17 00:00:00 2001 From: manjudr Date: Thu, 23 Dec 2021 12:53:04 +0530 Subject: [PATCH 2/8] SB-27924:feat: Refactor of CircleCI --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 86c34162c..3f8d5111c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ commands: export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 export PATH=$JAVA_HOME/bin:$PATH cd sunbird-core-dataproducts && mvn install -DskipTests - run-tests: + execute-tests: steps: - run: name: Build analytics-core-dataproducts dependency jar @@ -135,6 +135,6 @@ jobs: - image: "circleci/openjdk:11-jdk" steps: - checkout - - run-tests + - execute-tests From 8f6ab3d1b5352fe37cd8db9215a5e9f9f3da6852 Mon Sep 17 00:00:00 2001 From: manjudr Date: Thu, 23 Dec 2021 12:56:09 +0530 Subject: [PATCH 3/8] SB-27924:feat: Refactor of CircleCI --- .circleci/config.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f8d5111c..33a29c6a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,9 +46,9 @@ commands: - run: name: Build analytics-core-dataproducts dependency jar command: | - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 - export PATH=$JAVA_HOME/bin:$PATH - cd sunbird-core-dataproducts && mvn install -DskipTests + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 + export PATH=$JAVA_HOME/bin:$PATH + cd sunbird-core-dataproducts && mvn install -DskipTests execute-tests: steps: - run: @@ -136,5 +136,13 @@ jobs: steps: - checkout - execute-tests +workflows: + main: + jobs: + - build-artifacts + - run-tests: + requires: + - build-artifacts + From d2460d88828088ab49e747c265e0e273a6dcd650 Mon Sep 17 00:00:00 2001 From: manjudr Date: Thu, 23 Dec 2021 13:00:03 +0530 Subject: [PATCH 4/8] SB-27924:feat: CircleCI Validation Issue fix --- .circleci/config.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 33a29c6a8..0a7441f7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,4 @@ version: 2.1 - commands: clone-dependency-repos: steps: @@ -41,6 +40,7 @@ commands: export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 export PATH=$JAVA_HOME/bin:$PATH cd sunbird-analytics-core && mvn install -DskipTests + build-core-jobs-artifacts: steps: - run: @@ -49,14 +49,14 @@ commands: export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 export PATH=$JAVA_HOME/bin:$PATH cd sunbird-core-dataproducts && mvn install -DskipTests - execute-tests: - steps: - - run: - name: Build analytics-core-dataproducts dependency jar - command: | - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 - export PATH=$JAVA_HOME/bin:$PATH - cd data-products && mvn scoverage:report + execute-tests: + steps: + - run: + name: Build analytics-core-dataproducts dependency jar + command: | + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 + export PATH=$JAVA_HOME/bin:$PATH + cd data-products && mvn scoverage:report #jobs: # sunbird-ed-dp-build: # machine: From 35e33b9c5c8f1c32356cad674009457f04e046d4 Mon Sep 17 00:00:00 2001 From: manjudr Date: Thu, 23 Dec 2021 13:14:43 +0530 Subject: [PATCH 5/8] SB-27924:feat: CircleCI Validation Issue fix --- .circleci/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a7441f7c..387de59bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,11 +52,9 @@ commands: execute-tests: steps: - run: - name: Build analytics-core-dataproducts dependency jar + name: Execute the testcases. command: | - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 - export PATH=$JAVA_HOME/bin:$PATH - cd data-products && mvn scoverage:report + cd data-products && mvn -X sonar:sonar -Dsonar.projectKey=Sunbird-Ed_sunbird-data-products -Dsonar.host.url=https://sonarcloud.io -Dsonar.exclusions=/home/circleci/project/adhoc-scripts/**,/home/circleci/project/replay-scripts/** -Dsonar.scala.coverage.reportPaths=/home/circleci/project/data-products/target/scoverage.xml -Dsonar.organization=sunbird-ed #jobs: # sunbird-ed-dp-build: # machine: From 802f23cf5e7ba8d112eaa618b9ad26c36c1d1847 Mon Sep 17 00:00:00 2001 From: manjudr Date: Thu, 23 Dec 2021 13:30:56 +0530 Subject: [PATCH 6/8] SB-27924:feat: CircleCI Validation Issue fix --- .circleci/config.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 387de59bb..0393a588c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,9 @@ commands: - run: name: Execute the testcases. command: | - cd data-products && mvn -X sonar:sonar -Dsonar.projectKey=Sunbird-Ed_sunbird-data-products -Dsonar.host.url=https://sonarcloud.io -Dsonar.exclusions=/home/circleci/project/adhoc-scripts/**,/home/circleci/project/replay-scripts/** -Dsonar.scala.coverage.reportPaths=/home/circleci/project/data-products/target/scoverage.xml -Dsonar.organization=sunbird-ed + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 + export PATH=$JAVA_HOME/bin:$PATH + cd data-products && mvn scoverage:report #jobs: # sunbird-ed-dp-build: # machine: @@ -128,12 +130,18 @@ jobs: - clone-dependency-repos - build-framework-artifacts - build-core-jobs-artifacts + - persist_to_workspace: + root: . + paths: + - . run-tests: docker: - - image: "circleci/openjdk:11-jdk" + machine: true steps: - checkout - execute-tests + - attach_workspace: + at: . workflows: main: jobs: From 707e216a58456ac2edc9cb2a98e2f77e9101575b Mon Sep 17 00:00:00 2001 From: manjudr Date: Thu, 23 Dec 2021 13:58:29 +0530 Subject: [PATCH 7/8] SB-27924:feat: CircleCI Validation Issue fix --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0393a588c..baee8dc52 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,10 +138,11 @@ jobs: docker: machine: true steps: - - checkout - - execute-tests - attach_workspace: at: . + - checkout + - execute-tests + workflows: main: jobs: From 01887dc5cc0537510a8d4d7c55add39e15307267 Mon Sep 17 00:00:00 2001 From: manjudr Date: Thu, 23 Dec 2021 14:20:31 +0530 Subject: [PATCH 8/8] SB-27924:feat: CircleCI Validation Issue fix --- .circleci/config.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index baee8dc52..9a02df804 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -135,13 +135,20 @@ jobs: paths: - . run-tests: - docker: machine: true steps: - attach_workspace: at: . - - checkout + - checkout: + path: ~/project/sunbird-ed + - restore_cache: + keys: + - dp-dependency-cache-{{ checksum "data-products/pom.xml" }} - execute-tests + - save_cache: + key: dp-dependency-cache-{{ checksum "data-products/pom.xml" }} + paths: + - ~/.m2 workflows: main: