diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index 4b6a7ad00e46..4454bfd4d050 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -238,7 +238,7 @@ private WorkflowStats processWebAnalytics() { try { workflow.process(); } catch (SearchIndexException ex) { - jobData.setStatus(EventPublisherJob.Status.FAILED); + jobData.setStatus(EventPublisherJob.Status.RUNNING); jobData.setFailure(ex.getIndexingError()); } @@ -252,7 +252,7 @@ private WorkflowStats processCostAnalysis() { try { workflow.process(); } catch (SearchIndexException ex) { - jobData.setStatus(EventPublisherJob.Status.FAILED); + jobData.setStatus(EventPublisherJob.Status.RUNNING); jobData.setFailure(ex.getIndexingError()); } @@ -268,7 +268,7 @@ private WorkflowStats processDataAssets() { try { workflow.process(); } catch (SearchIndexException ex) { - jobData.setStatus(EventPublisherJob.Status.FAILED); + jobData.setStatus(EventPublisherJob.Status.RUNNING); jobData.setFailure(ex.getIndexingError()); } @@ -282,13 +282,32 @@ private void updateJobStatsWithWorkflowStats(WorkflowStats workflowStats) { updateStats(stepName, stats); } } + private boolean isPartialSuccess(){ + try{ + if (jobData.getStats() != null && jobData.getStats().getJobStats() != null){ + float failure = jobData.getStats().getJobStats().getFailedRecords(); + float success = jobData.getStats().getJobStats().getSuccessRecords(); + // if success % is >= 90% + if (success/(success+failure) >= 0.9) return true; + } + } + catch (ArithmeticException ex){ + // ignore in case there are 0 success and failures + } + return false; + } private void updateJobStatus() { if (stopped) { jobData.setStatus(EventPublisherJob.Status.STOPPED); } else { if (jobData.getFailure() != null) { - jobData.setStatus(EventPublisherJob.Status.FAILED); + if (isPartialSuccess()){ + jobData.setStatus(EventPublisherJob.Status.PARTIAL_SUCCESS); + } + else { + jobData.setStatus(EventPublisherJob.Status.FAILED); + } } else { jobData.setStatus(EventPublisherJob.Status.COMPLETED); } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json index 9ef46dcb9b30..51b616128aa9 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json @@ -21,7 +21,8 @@ "active", "activeError", "stopped", - "success" + "success", + "partialSuccess" ] }, "runType": { diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json index e6f6b9922975..712b69971b25 100644 --- a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json +++ b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json @@ -75,7 +75,8 @@ "active", "activeError", "stopped", - "success" + "success", + "partialSuccess" ] }, "failure": {