From 840f06cf707dd0b1ee68a1e72f64ac5637d334dc Mon Sep 17 00:00:00 2001
From: ulixius9 <mayursingal9@gmail.com>
Date: Thu, 26 Sep 2024 20:14:17 +0530
Subject: [PATCH] MINOR: DI APP Support Partial Success

---
 .../bundles/insights/DataInsightsApp.java     | 27 ++++++++++++++++---
 .../entity/applications/appRunRecord.json     |  3 ++-
 .../json/schema/system/eventPublisherJob.json |  3 ++-
 3 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
index 4b6a7ad00e46..4454bfd4d050 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
@@ -238,7 +238,7 @@ private WorkflowStats processWebAnalytics() {
     try {
       workflow.process();
     } catch (SearchIndexException ex) {
-      jobData.setStatus(EventPublisherJob.Status.FAILED);
+      jobData.setStatus(EventPublisherJob.Status.RUNNING);
       jobData.setFailure(ex.getIndexingError());
     }
 
@@ -252,7 +252,7 @@ private WorkflowStats processCostAnalysis() {
     try {
       workflow.process();
     } catch (SearchIndexException ex) {
-      jobData.setStatus(EventPublisherJob.Status.FAILED);
+      jobData.setStatus(EventPublisherJob.Status.RUNNING);
       jobData.setFailure(ex.getIndexingError());
     }
 
@@ -268,7 +268,7 @@ private WorkflowStats processDataAssets() {
     try {
       workflow.process();
     } catch (SearchIndexException ex) {
-      jobData.setStatus(EventPublisherJob.Status.FAILED);
+      jobData.setStatus(EventPublisherJob.Status.RUNNING);
       jobData.setFailure(ex.getIndexingError());
     }
 
@@ -282,13 +282,32 @@ private void updateJobStatsWithWorkflowStats(WorkflowStats workflowStats) {
       updateStats(stepName, stats);
     }
   }
+  private boolean isPartialSuccess(){
+    try{
+      if (jobData.getStats() != null && jobData.getStats().getJobStats() != null){
+        float failure = jobData.getStats().getJobStats().getFailedRecords();
+        float success = jobData.getStats().getJobStats().getSuccessRecords();
+        // if success % is >= 90%
+        if (success/(success+failure) >= 0.9) return true;
+      }
+    }
+    catch (ArithmeticException ex){
+      // ignore in case there are 0 success and failures
+    }
+    return false;
+  }
 
   private void updateJobStatus() {
     if (stopped) {
       jobData.setStatus(EventPublisherJob.Status.STOPPED);
     } else {
       if (jobData.getFailure() != null) {
-        jobData.setStatus(EventPublisherJob.Status.FAILED);
+        if (isPartialSuccess()){
+          jobData.setStatus(EventPublisherJob.Status.PARTIAL_SUCCESS);
+        }
+        else {
+          jobData.setStatus(EventPublisherJob.Status.FAILED);
+        }
       } else {
         jobData.setStatus(EventPublisherJob.Status.COMPLETED);
       }
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json
index 9ef46dcb9b30..51b616128aa9 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json
@@ -21,7 +21,8 @@
         "active",
         "activeError",
         "stopped",
-        "success"
+        "success",
+        "partialSuccess"
       ]
     },
     "runType": {
diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
index e6f6b9922975..712b69971b25 100644
--- a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
+++ b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
@@ -75,7 +75,8 @@
         "active",
         "activeError",
         "stopped",
-        "success"
+        "success",
+        "partialSuccess"
       ]
     },
     "failure": {