MINOR: DI APP Support Partial Success

open-metadata · Sep 26, 2024 · 840f06c · 840f06c
1 parent d264495
commit 840f06c
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 6 deletions.
diff --git a/...service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/...service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
@@ -238,7 +238,7 @@ private WorkflowStats processWebAnalytics() {
  try {
  workflow.process();
  } catch (SearchIndexException ex) {
- jobData.setStatus(EventPublisherJob.Status.FAILED);
+ jobData.setStatus(EventPublisherJob.Status.RUNNING);
  jobData.setFailure(ex.getIndexingError());
  }
 
@@ -252,7 +252,7 @@ private WorkflowStats processCostAnalysis() {
  try {
  workflow.process();
  } catch (SearchIndexException ex) {
- jobData.setStatus(EventPublisherJob.Status.FAILED);
+ jobData.setStatus(EventPublisherJob.Status.RUNNING);
  jobData.setFailure(ex.getIndexingError());
  }
 
@@ -268,7 +268,7 @@ private WorkflowStats processDataAssets() {
  try {
  workflow.process();
  } catch (SearchIndexException ex) {
- jobData.setStatus(EventPublisherJob.Status.FAILED);
+ jobData.setStatus(EventPublisherJob.Status.RUNNING);
  jobData.setFailure(ex.getIndexingError());
  }
 
@@ -282,13 +282,32 @@ private void updateJobStatsWithWorkflowStats(WorkflowStats workflowStats) {
  updateStats(stepName, stats);
  }
  }
+ private boolean isPartialSuccess(){
+ try{
+ if (jobData.getStats() != null && jobData.getStats().getJobStats() != null){
+ float failure = jobData.getStats().getJobStats().getFailedRecords();
+ float success = jobData.getStats().getJobStats().getSuccessRecords();
+ // if success % is >= 90%
+ if (success/(success+failure) >= 0.9) return true;
+ }
+ }
+ catch (ArithmeticException ex){
+ // ignore in case there are 0 success and failures
+ }
+ return false;
+ }
 
  private void updateJobStatus() {
  if (stopped) {
  jobData.setStatus(EventPublisherJob.Status.STOPPED);
  } else {
  if (jobData.getFailure() != null) {
- jobData.setStatus(EventPublisherJob.Status.FAILED);
+ if (isPartialSuccess()){
+ jobData.setStatus(EventPublisherJob.Status.PARTIAL_SUCCESS);
+ }
+ else {
+ jobData.setStatus(EventPublisherJob.Status.FAILED);
+ }
  } else {
  jobData.setStatus(EventPublisherJob.Status.COMPLETED);
  }

diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json
@@ -21,7 +21,8 @@
  "active",
  "activeError",
  "stopped",
- "success"
+ "success",
+ "partialSuccess"
  ]
  },
  "runType": {

diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
@@ -75,7 +75,8 @@
  "active",
  "activeError",
  "stopped",
- "success"
+ "success",
+ "partialSuccess"
  ]
  },
  "failure": {