From 840f06cf707dd0b1ee68a1e72f64ac5637d334dc Mon Sep 17 00:00:00 2001 From: ulixius9 Date: Thu, 26 Sep 2024 20:14:17 +0530 Subject: [PATCH 1/3] MINOR: DI APP Support Partial Success --- .../bundles/insights/DataInsightsApp.java | 27 ++++++++++++++++--- .../entity/applications/appRunRecord.json | 3 ++- .../json/schema/system/eventPublisherJob.json | 3 ++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index 4b6a7ad00e46..4454bfd4d050 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -238,7 +238,7 @@ private WorkflowStats processWebAnalytics() { try { workflow.process(); } catch (SearchIndexException ex) { - jobData.setStatus(EventPublisherJob.Status.FAILED); + jobData.setStatus(EventPublisherJob.Status.RUNNING); jobData.setFailure(ex.getIndexingError()); } @@ -252,7 +252,7 @@ private WorkflowStats processCostAnalysis() { try { workflow.process(); } catch (SearchIndexException ex) { - jobData.setStatus(EventPublisherJob.Status.FAILED); + jobData.setStatus(EventPublisherJob.Status.RUNNING); jobData.setFailure(ex.getIndexingError()); } @@ -268,7 +268,7 @@ private WorkflowStats processDataAssets() { try { workflow.process(); } catch (SearchIndexException ex) { - jobData.setStatus(EventPublisherJob.Status.FAILED); + jobData.setStatus(EventPublisherJob.Status.RUNNING); jobData.setFailure(ex.getIndexingError()); } @@ -282,13 +282,32 @@ private void updateJobStatsWithWorkflowStats(WorkflowStats workflowStats) { updateStats(stepName, stats); } } + private boolean isPartialSuccess(){ + try{ + if (jobData.getStats() != null && jobData.getStats().getJobStats() != null){ + float failure = jobData.getStats().getJobStats().getFailedRecords(); + float success = jobData.getStats().getJobStats().getSuccessRecords(); + // if success % is >= 90% + if (success/(success+failure) >= 0.9) return true; + } + } + catch (ArithmeticException ex){ + // ignore in case there are 0 success and failures + } + return false; + } private void updateJobStatus() { if (stopped) { jobData.setStatus(EventPublisherJob.Status.STOPPED); } else { if (jobData.getFailure() != null) { - jobData.setStatus(EventPublisherJob.Status.FAILED); + if (isPartialSuccess()){ + jobData.setStatus(EventPublisherJob.Status.PARTIAL_SUCCESS); + } + else { + jobData.setStatus(EventPublisherJob.Status.FAILED); + } } else { jobData.setStatus(EventPublisherJob.Status.COMPLETED); } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json index 9ef46dcb9b30..51b616128aa9 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json @@ -21,7 +21,8 @@ "active", "activeError", "stopped", - "success" + "success", + "partialSuccess" ] }, "runType": { diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json index e6f6b9922975..712b69971b25 100644 --- a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json +++ b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json @@ -75,7 +75,8 @@ "active", "activeError", "stopped", - "success" + "success", + "partialSuccess" ] }, "failure": { From 9a53b486d84419c2af85586d4b0d5ea75e4a06b2 Mon Sep 17 00:00:00 2001 From: ulixius9 Date: Thu, 26 Sep 2024 21:04:56 +0530 Subject: [PATCH 2/3] add partial status support --- .../service/apps/scheduler/AbstractOmAppJobListener.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java index 8e23e7be7f40..939b9270f6e2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java @@ -94,7 +94,10 @@ public void jobWasExecuted( if (jobException == null && !(runRecord.getStatus() == AppRunRecord.Status.FAILED || runRecord.getStatus() == AppRunRecord.Status.ACTIVE_ERROR)) { - runRecord.withStatus(AppRunRecord.Status.SUCCESS); + runRecord.withStatus( + runRecord.getStatus() == AppRunRecord.Status.PARTIAL_SUCCESS + ? AppRunRecord.Status.PARTIAL_SUCCESS + : AppRunRecord.Status.SUCCESS); SuccessContext context = new SuccessContext(); if (runRecord.getSuccessContext() != null) { context = runRecord.getSuccessContext(); From 57bac43e1980209ae5181bf0da31ced7fd1b88b7 Mon Sep 17 00:00:00 2001 From: ulixius9 Date: Fri, 27 Sep 2024 12:24:53 +0530 Subject: [PATCH 3/3] format --- .../apps/bundles/insights/DataInsightsApp.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index 4454bfd4d050..e9ddb9430842 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -282,16 +282,16 @@ private void updateJobStatsWithWorkflowStats(WorkflowStats workflowStats) { updateStats(stepName, stats); } } - private boolean isPartialSuccess(){ - try{ - if (jobData.getStats() != null && jobData.getStats().getJobStats() != null){ + + private boolean isPartialSuccess() { + try { + if (jobData.getStats() != null && jobData.getStats().getJobStats() != null) { float failure = jobData.getStats().getJobStats().getFailedRecords(); float success = jobData.getStats().getJobStats().getSuccessRecords(); // if success % is >= 90% - if (success/(success+failure) >= 0.9) return true; + if (success / (success + failure) >= 0.9) return true; } - } - catch (ArithmeticException ex){ + } catch (ArithmeticException ex) { // ignore in case there are 0 success and failures } return false; @@ -302,10 +302,9 @@ private void updateJobStatus() { jobData.setStatus(EventPublisherJob.Status.STOPPED); } else { if (jobData.getFailure() != null) { - if (isPartialSuccess()){ + if (isPartialSuccess()) { jobData.setStatus(EventPublisherJob.Status.PARTIAL_SUCCESS); - } - else { + } else { jobData.setStatus(EventPublisherJob.Status.FAILED); } } else {