Include exit codes in job failed event. (#282)

armadaproject · Dec 16, 2019 · 1365f1c · 1365f1c
1 parent a7a6f9e
commit 1365f1c
Show file tree

Hide file tree

Showing 9 changed files with 253 additions and 69 deletions.
diff --git a/client/DotNet/Armada.Client/ClientGenerated.cs b/client/DotNet/Armada.Client/ClientGenerated.cs
@@ -594,6 +594,9 @@ public partial class ApiJobFailedEvent
         [Newtonsoft.Json.JsonProperty("Created", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public System.DateTimeOffset? Created { get; set; }
 
+        [Newtonsoft.Json.JsonProperty("ExitCodes", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IDictionary<string, int> ExitCodes { get; set; }
+
         [Newtonsoft.Json.JsonProperty("JobId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public string JobId { get; set; }
 

diff --git a/internal/armada/api/api.swagger.go b/internal/armada/api/api.swagger.go
@@ -316,6 +316,13 @@ func SwaggerJsonTemplate() string {
 		"          \"type\": \"string\",\n" +
 		"          \"format\": \"date-time\"\n" +
 		"        },\n" +
+		"        \"ExitCodes\": {\n" +
+		"          \"type\": \"object\",\n" +
+		"          \"additionalProperties\": {\n" +
+		"            \"type\": \"integer\",\n" +
+		"            \"format\": \"int32\"\n" +
+		"          }\n" +
+		"        },\n" +
 		"        \"JobId\": {\n" +
 		"          \"type\": \"string\"\n" +
 		"        },\n" +

diff --git a/internal/armada/api/api.swagger.json b/internal/armada/api/api.swagger.json
@@ -305,6 +305,13 @@
           "type": "string",
           "format": "date-time"
         },
+        "ExitCodes": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "integer",
+            "format": "int32"
+          }
+        },
         "JobId": {
           "type": "string"
         },

diff --git a/internal/armada/api/event.pb.go b/internal/armada/api/event.pb.go
diff --git a/internal/armada/api/event.proto b/internal/armada/api/event.proto
@@ -79,6 +79,7 @@ message JobFailedEvent {
     google.protobuf.Timestamp Created = 4 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
     string ClusterId = 5;
     string Reason = 6;
+    map<string, int32> ExitCodes = 7;
 }
 
 message JobSucceededEvent {

diff --git a/internal/executor/reporter/event.go b/internal/executor/reporter/event.go
@@ -33,7 +33,7 @@ func CreateEventForCurrentState(pod *v1.Pod, clusterId string) (api.Event, error
 			ClusterId: clusterId,
 		}, nil
 	case v1.PodFailed:
-		return CreateJobFailedEvent(pod, util.ExtractPodFailedReason(pod), clusterId), nil
+		return CreateJobFailedEvent(pod, util.ExtractPodFailedReason(pod), util.ExtractPodExitCodes(pod), clusterId), nil
 	case v1.PodSucceeded:
 		return &api.JobSucceededEvent{
 			JobId:     pod.Labels[domain.JobId],
@@ -69,13 +69,14 @@ func CreateJobLeaseReturnedEvent(pod *v1.Pod, reason string, clusterId string) a
 	}
 }
 
-func CreateJobFailedEvent(pod *v1.Pod, reason string, clusterId string) api.Event {
+func CreateJobFailedEvent(pod *v1.Pod, reason string, exitCodes map[string]int32, clusterId string) api.Event {
 	return &api.JobFailedEvent{
 		JobId:     pod.Labels[domain.JobId],
 		JobSetId:  pod.Annotations[domain.JobSetId],
 		Queue:     pod.Labels[domain.Queue],
 		Created:   time.Now(),
 		ClusterId: clusterId,
 		Reason:    reason,
+		ExitCodes: exitCodes,
 	}
 }
diff --git a/internal/executor/service/cluster_allocation.go b/internal/executor/service/cluster_allocation.go
@@ -108,7 +108,7 @@ func (allocationService *ClusterAllocationService) failJobs(failedSubmissions []
 	toBeReportedDone := make([]*v1.Pod, 0, 10)
 
 	for _, details := range failedSubmissions {
-		failEvent := reporter.CreateJobFailedEvent(details.pod, details.error.Status().Message, allocationService.clusterContext.GetClusterId())
+		failEvent := reporter.CreateJobFailedEvent(details.pod, details.error.Status().Message, map[string]int32{}, allocationService.clusterContext.GetClusterId())
 		err := allocationService.eventReporter.Report(failEvent)
 
 		if err == nil {

diff --git a/internal/executor/service/stuck_pod_detector.go b/internal/executor/service/stuck_pod_detector.go
@@ -40,7 +40,7 @@ func (podProgressMonitor *StuckPodDetector) onStuckPodDetected(pod *v1.Pod) (res
 	if util.IsRetryable(pod) {
 		event = reporter.CreateJobUnableToScheduleEvent(pod, util.ExtractPodStuckReason(pod), podProgressMonitor.clusterId)
 	} else {
-		event = reporter.CreateJobFailedEvent(pod, util.ExtractPodStuckReason(pod), podProgressMonitor.clusterId)
+		event = reporter.CreateJobFailedEvent(pod, util.ExtractPodStuckReason(pod), map[string]int32{}, podProgressMonitor.clusterId)
 	}
 
 	err := podProgressMonitor.eventReporter.Report(event)

diff --git a/internal/executor/util/pod_status.go b/internal/executor/util/pod_status.go
@@ -44,6 +44,21 @@ func ExtractPodFailedReason(pod *v1.Pod) string {
 	return failedMessage
 }
 
+func ExtractPodExitCodes(pod *v1.Pod) map[string]int32 {
+	containerStatuses := pod.Status.ContainerStatuses
+	containerStatuses = append(containerStatuses, pod.Status.InitContainerStatuses...)
+
+	exitCodes := map[string]int32{}
+
+	for _, containerStatus := range containerStatuses {
+		if containerStatus.State.Terminated != nil {
+			exitCodes[containerStatus.Name] = containerStatus.State.Terminated.ExitCode
+		}
+	}
+
+	return exitCodes
+}
+
 func IsRetryable(pod *v1.Pod) bool {
 	containerStatuses := pod.Status.ContainerStatuses
 	containerStatuses = append(containerStatuses, pod.Status.InitContainerStatuses...)