From 7dec5026d11b4866425d61ac8cbdb2b80247f9bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nils=20Gustav=20Str=C3=A5b=C3=B8?= Date: Wed, 3 Jan 2024 16:00:10 +0100 Subject: [PATCH] remove legacy job handler --- .../environment_controller_secrets_test.go | 1 - api/environments/job_handler.go | 49 +-- api/environments/legacy_job_handler.go | 346 ------------------ api/environments/legacy_job_status.go | 167 --------- 4 files changed, 3 insertions(+), 560 deletions(-) delete mode 100644 api/environments/legacy_job_handler.go delete mode 100644 api/environments/legacy_job_status.go diff --git a/api/environments/environment_controller_secrets_test.go b/api/environments/environment_controller_secrets_test.go index a338a7ad..fc009d74 100644 --- a/api/environments/environment_controller_secrets_test.go +++ b/api/environments/environment_controller_secrets_test.go @@ -1309,7 +1309,6 @@ func (s *secretHandlerTestSuite) createPodForRadixComponent(kubeClient kubernete kube.RadixComponentLabel: componentName, } if isJobComponent { - labels[k8sJobNameLabel] = componentName labels[kube.RadixJobTypeLabel] = kube.RadixJobTypeJobSchedule } _, _ = kubeClient.CoreV1().Pods(envNamespace).Create(context.Background(), &corev1.Pod{ diff --git a/api/environments/job_handler.go b/api/environments/job_handler.go index 6913ed14..36909e45 100644 --- a/api/environments/job_handler.go +++ b/api/environments/job_handler.go @@ -35,15 +35,6 @@ func (eh EnvironmentHandler) GetJobs(ctx context.Context, appName, envName, jobC return nil, err } - // Backward compatibility: Get list of jobs not handled by RadixBatch - // TODO: Remove when there are no legacy jobs left - jh := legacyJobHandler{accounts: eh.accounts} - legacyJobs, err := jh.GetJobs(ctx, appName, envName, jobComponentName) - if err != nil { - return nil, err - } - jobs = append(jobs, legacyJobs...) - sort.SliceStable(jobs, func(i, j int) bool { return utils.IsBefore(&jobs[j], &jobs[i]) }) @@ -62,16 +53,7 @@ func (eh EnvironmentHandler) getJobs(ctx context.Context, appName, envName, jobC // GetJob Gets job by name func (eh EnvironmentHandler) GetJob(ctx context.Context, appName, envName, jobComponentName, jobName string) (*deploymentModels.ScheduledJobSummary, error) { - if jobSummary, err := eh.getJob(ctx, appName, envName, jobComponentName, jobName); err == nil { - return jobSummary, nil - } - - // TODO: Return error from getJob when legacy handler is removed - // TODO: Remove when there are no legacy jobs left - - // Backward compatibility: Get job not handled by RadixBatch - jh := legacyJobHandler{accounts: eh.accounts} - return jh.GetJob(ctx, appName, envName, jobComponentName, jobName) + return eh.getJob(ctx, appName, envName, jobComponentName, jobName) } // StopJob Stop job by name @@ -178,15 +160,6 @@ func (eh EnvironmentHandler) GetBatches(ctx context.Context, appName, envName, j return nil, err } - // Backward compatibility: Get list of batches not handled by RadixBatch - // TODO: Remove when there are no legacy jobs left - jh := legacyJobHandler{accounts: eh.accounts} - legacyBatches, err := jh.GetBatches(ctx, appName, envName, jobComponentName) - if err != nil { - return nil, err - } - summaries = append(summaries, legacyBatches...) - sort.SliceStable(summaries, func(i, j int) bool { return utils.IsBefore(&summaries[j], &summaries[i]) }) @@ -265,16 +238,7 @@ func (eh EnvironmentHandler) CopyJob(ctx context.Context, appName, envName, jobC // GetBatch Gets batch by name func (eh EnvironmentHandler) GetBatch(ctx context.Context, appName, envName, jobComponentName, batchName string) (*deploymentModels.ScheduledBatchSummary, error) { - if batchSummary, err := eh.getBatch(ctx, appName, envName, jobComponentName, batchName); err == nil { - return batchSummary, nil - } - - // TODO: Return error from getBatch when legacy handler is removed - // TODO: Remove legacy handler when there are no legacy jobs left - - // Backward compatibility: Get batch not handled by RadixBatch - jh := legacyJobHandler{accounts: eh.accounts} - return jh.GetBatch(ctx, appName, envName, jobComponentName, batchName) + return eh.getBatch(ctx, appName, envName, jobComponentName, batchName) } func (eh EnvironmentHandler) getBatch(ctx context.Context, appName, envName, jobComponentName, batchName string) (*deploymentModels.ScheduledBatchSummary, error) { @@ -295,14 +259,7 @@ func (eh EnvironmentHandler) getBatch(ctx context.Context, appName, envName, job // GetJobPayload Gets job payload func (eh EnvironmentHandler) GetJobPayload(ctx context.Context, appName, envName, jobComponentName, jobName string) (io.ReadCloser, error) { - if payload, err := eh.getJobPayload(ctx, appName, envName, jobComponentName, jobName); err == nil { - return payload, nil - } - - // Backward compatibility: Get batch not handled by RadixBatch - // TODO: Remove when there are no legacy jobs left - jh := legacyJobHandler{accounts: eh.accounts} - return jh.GetJobPayload(appName, envName, jobComponentName, jobName) + return eh.getJobPayload(ctx, appName, envName, jobComponentName, jobName) } func (eh EnvironmentHandler) getJobPayload(ctx context.Context, appName, envName, jobComponentName, jobName string) (io.ReadCloser, error) { diff --git a/api/environments/legacy_job_handler.go b/api/environments/legacy_job_handler.go deleted file mode 100644 index 990e1483..00000000 --- a/api/environments/legacy_job_handler.go +++ /dev/null @@ -1,346 +0,0 @@ -package environments - -import ( - "bytes" - "context" - "fmt" - "io" - "sort" - "strconv" - "strings" - - deploymentModels "github.com/equinor/radix-api/api/deployments/models" - environmentModels "github.com/equinor/radix-api/api/environments/models" - jobModels "github.com/equinor/radix-api/api/jobs/models" - "github.com/equinor/radix-api/api/utils" - apiModels "github.com/equinor/radix-api/models" - radixutils "github.com/equinor/radix-common/utils" - - // batchSchedulerApi "github.com/equinor/radix-job-scheduler/api/batches" - // jobSchedulerApi "github.com/equinor/radix-job-scheduler/api/jobs" - - "github.com/equinor/radix-operator/pkg/apis/kube" - operatorUtils "github.com/equinor/radix-operator/pkg/apis/utils" - log "github.com/sirupsen/logrus" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" -) - -const ( - legacyRadixBatchJobCountAnnotation = "radix.equinor.com/batch-job-count" - legacyJobPayloadPropertyName = "payload" - k8sJobNameLabel = "job-name" // A label that k8s automatically adds to a Pod created by a Job -) - -type legacyJobHandler struct { - accounts apiModels.Accounts -} - -// GetJobs Get jobs -func (h legacyJobHandler) GetJobs(ctx context.Context, appName, envName, jobComponentName string) ([]deploymentModels. -ScheduledJobSummary, error) { - namespace := operatorUtils.GetEnvironmentNamespace(appName, envName) - jobs, err := h.getSingleJobs(ctx, namespace, jobComponentName) - if err != nil { - return nil, err - } - jobPodLabelSelector := labels.Set{ - kube.RadixJobTypeLabel: kube.RadixJobTypeJobSchedule, - } - podList, err := h.getPodsForSelector(ctx, namespace, labels.SelectorFromSet(jobPodLabelSelector)) - if err != nil { - return nil, err - } - jobPodMap, err := h.getJobPodsMap(podList) - if err != nil { - return nil, err - } - jobSummaryList := h.getScheduledJobSummaryList(jobs, jobPodMap) - return jobSummaryList, nil -} - -func (h legacyJobHandler) GetJob(ctx context.Context, appName, envName, jobComponentName, jobName string) (*deploymentModels. -ScheduledJobSummary, error) { - namespace := operatorUtils.GetEnvironmentNamespace(appName, envName) - job, err := h.getJob(ctx, namespace, jobComponentName, jobName, kube.RadixJobTypeJobSchedule) - if err != nil { - return nil, err - } - jobPodLabelSelector := labels.Set{ - k8sJobNameLabel: jobName, - kube.RadixJobTypeLabel: kube.RadixJobTypeJobSchedule, - } - podList, err := h.getPodsForSelector(ctx, namespace, labels.SelectorFromSet(jobPodLabelSelector)) - if err != nil { - return nil, err - } - jobPodMap, err := h.getJobPodsMap(podList) - if err != nil { - return nil, err - } - jobSummary := h.getScheduledJobSummary(job, jobPodMap) - return jobSummary, nil -} - -// GetBatches Get batches -func (h legacyJobHandler) GetBatches(ctx context.Context, appName, envName, jobComponentName string) ([]deploymentModels. -ScheduledBatchSummary, error) { - namespace := operatorUtils.GetEnvironmentNamespace(appName, envName) - batches, err := h.getBatches(ctx, namespace, jobComponentName) - if err != nil { - return nil, err - } - return h.getScheduledBatchSummaryList(batches) -} - -func (h legacyJobHandler) getScheduledJobSummaryList(jobs []batchv1.Job, - jobPodsMap map[string][]corev1.Pod) []deploymentModels.ScheduledJobSummary { - summaries := make([]deploymentModels.ScheduledJobSummary, 0) // return an array - not null - for _, job := range jobs { - summary := h.getScheduledJobSummary(&job, jobPodsMap) - summaries = append(summaries, *summary) - } - - // Sort job-summaries descending - sort.Slice(summaries, func(i, j int) bool { - return utils.IsBefore(&summaries[j], &summaries[i]) - }) - return summaries -} - -func (h legacyJobHandler) GetBatch(ctx context.Context, appName, envName, jobComponentName, batchName string) (*deploymentModels. -ScheduledBatchSummary, error) { - namespace := operatorUtils.GetEnvironmentNamespace(appName, envName) - batch, err := h.getJob(ctx, namespace, jobComponentName, batchName, kube.RadixJobTypeBatchSchedule) - if err != nil { - return nil, err - } - summary, err := h.getScheduledBatchSummary(batch) - if err != nil { - return nil, err - } - kubeClient := h.accounts.UserAccount.Client - jobPodLabelSelector := labels.Set{ - kube.RadixBatchNameLabel: batchName, - } - batchPods, err := h.getPodsForSelector(ctx, namespace, labels.SelectorFromSet(jobPodLabelSelector)) - if err != nil { - return nil, err - } - batchStatus, err := GetBatchStatusFromJob(kubeClient, batch, batchPods) - if err != nil { - return nil, err - } - summary.Status = batchStatus.Status - //nolint:staticcheck // SA1019 support old batch scheduler - summary.Message = batchStatus.Message - - jobPodsMap, err := h.getJobPodsMap(batchPods) - if err != nil { - return nil, err - } - if batchPod, ok := jobPodsMap[batchName]; ok && len(batchPod) > 0 { - batchPodSummary := deploymentModels.GetReplicaSummary(batchPod[0]) - //nolint:staticcheck // SA1019 support old batch scheduler - summary.Replica = &batchPodSummary - } - batchJobSummaryList, err := h.getBatchJobSummaryList(ctx, namespace, jobComponentName, batchName, jobPodsMap) - if err != nil { - return nil, err - } - summary.JobList = batchJobSummaryList - return summary, nil -} - -// GetJobPayload Gets job payload -func (h legacyJobHandler) GetJobPayload(appName, envName, jobComponentName, jobName string) (io.ReadCloser, error) { - namespace := operatorUtils.GetEnvironmentNamespace(appName, envName) - kubeUtil, err := kube.New(h.accounts.ServiceAccount.Client, h.accounts.UserAccount.RadixClient, nil) - if err != nil { - return nil, err - } - payloadSecrets, err := kubeUtil.ListSecretsWithSelector(namespace, h.getJobsSchedulerPayloadSecretSelector(appName, jobComponentName, jobName)) - if err != nil { - return nil, err - } - if len(payloadSecrets) == 0 { - return nil, environmentModels.ScheduledJobPayloadNotFoundError(appName, jobName) - } - if len(payloadSecrets) > 1 { - return nil, environmentModels.ScheduledJobPayloadUnexpectedError(appName, jobName, "unexpected multiple payloads found") - } - payload := payloadSecrets[0].Data[legacyJobPayloadPropertyName] - return io.NopCloser(bytes.NewReader(payload)), nil -} - -func (h legacyJobHandler) getJobsSchedulerPayloadSecretSelector(appName, jobComponentName, jobName string) string { - return labels.SelectorFromSet(map[string]string{ - kube.RadixAppLabel: appName, - kube.RadixComponentLabel: jobComponentName, - kube.RadixJobTypeLabel: kube.RadixJobTypeJobSchedule, - kube.RadixJobNameLabel: jobName, - }).String() -} - -func (h legacyJobHandler) getScheduledJobSummary(job *batchv1.Job, - jobPodsMap map[string][]corev1.Pod) *deploymentModels.ScheduledJobSummary { - creationTimestamp := job.GetCreationTimestamp() - batchName := job.ObjectMeta.Labels[kube.RadixBatchNameLabel] - summary := deploymentModels.ScheduledJobSummary{ - Name: job.Name, - Created: radixutils.FormatTimestamp(creationTimestamp.Time), - Started: radixutils.FormatTime(job.Status.StartTime), - BatchName: batchName, - JobId: job.ObjectMeta.Labels["radix-job-id"], - } - summary.TimeLimitSeconds = job.Spec.Template.Spec.ActiveDeadlineSeconds - jobPods := jobPodsMap[job.Name] - if len(jobPods) > 0 { - summary.ReplicaList = getReplicaSummariesForPods(jobPods) - } - summary.Resources = h.getJobResourceRequirements(job, jobPods) - summary.BackoffLimit = h.getJobBackoffLimit(job) - jobStatus := GetJobStatusFromJob(h.accounts.UserAccount.Client, job, jobPodsMap[job.Name]) - summary.Status = jobStatus.Status - summary.Message = jobStatus.Message - summary.Ended = jobStatus.Ended - return &summary -} - -func (h legacyJobHandler) getScheduledBatchSummaryList(batches []batchv1.Job) ([]deploymentModels.ScheduledBatchSummary, error) { - summaries := make([]deploymentModels.ScheduledBatchSummary, 0) // return an array - not null - for _, batch := range batches { - summary, err := h.getScheduledBatchSummary(&batch) - if err != nil { - return nil, err - } - summary.Status = jobModels.Succeeded.String() // TODO should be real status? - summaries = append(summaries, *summary) - } - - return summaries, nil -} - -func (h legacyJobHandler) getScheduledBatchSummary(batch *batchv1.Job) (*deploymentModels.ScheduledBatchSummary, error) { - creationTimestamp := batch.GetCreationTimestamp() - summary := deploymentModels.ScheduledBatchSummary{ - Name: batch.Name, - Created: radixutils.FormatTimestamp(creationTimestamp.Time), - Started: radixutils.FormatTime(batch.Status.StartTime), - Ended: radixutils.FormatTime(batch.Status.CompletionTime), - } - if jobCount, ok := batch.ObjectMeta.Annotations[legacyRadixBatchJobCountAnnotation]; ok { - if count, err := strconv.Atoi(jobCount); err == nil { - summary.TotalJobCount = count - } else { - log.Warnf("failed to get job count for the annotation %s", - legacyRadixBatchJobCountAnnotation) - } - } - - return &summary, nil -} - -func (h legacyJobHandler) getBatchJobSummaryList(ctx context.Context, namespace string, jobComponentName string, batchName string, jobPodsMap map[string][]corev1.Pod) ([]deploymentModels.ScheduledJobSummary, error) { - summaries := make([]deploymentModels.ScheduledJobSummary, 0) // return an array - not null - batchJobs, err := h.getBatchJobs(ctx, namespace, jobComponentName, batchName) - if err != nil { - return nil, err - } - for _, job := range batchJobs { - summaries = append(summaries, *h.getScheduledJobSummary(&job, jobPodsMap)) - } - return summaries, nil -} - -func (h legacyJobHandler) getJobBackoffLimit(job *batchv1.Job) int32 { - if job.Spec.BackoffLimit == nil { - return 0 - } - return *job.Spec.BackoffLimit -} - -func (h legacyJobHandler) getJobResourceRequirements(job *batchv1.Job, jobPods []corev1.Pod) deploymentModels.ResourceRequirements { - if len(jobPods) > 0 && len(jobPods[0].Spec.Containers) > 0 { - return deploymentModels.ConvertResourceRequirements(jobPods[0].Spec.Containers[0].Resources) - } else if len(job.Spec.Template.Spec.Containers) > 0 { - return deploymentModels.ConvertResourceRequirements(job.Spec.Template.Spec.Containers[0].Resources) - } - return deploymentModels.ResourceRequirements{} -} - -func (h legacyJobHandler) getJobPodsMap(podList []corev1.Pod) (map[string][]corev1.Pod, error) { - jobPodMap := make(map[string][]corev1.Pod) - for _, pod := range podList { - pod := pod - if jobName, ok := pod.GetLabels()[k8sJobNameLabel]; ok { - jobPodList := jobPodMap[jobName] - jobPodMap[jobName] = append(jobPodList, pod) - } - } - return jobPodMap, nil -} - -func (h legacyJobHandler) getPodsForSelector(ctx context.Context, namespace string, selector labels.Selector) ([]corev1.Pod, error) { - podList, err := h.accounts.UserAccount.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: selector.String(), - }) - if err != nil { - return nil, err - } - return podList.Items, err -} - -func (h legacyJobHandler) getSingleJobs(ctx context.Context, namespace, componentName string) ([]batchv1.Job, error) { - batchNameNotExistsRequirement, err := labels.NewRequirement(kube.RadixBatchNameLabel, selection.DoesNotExist, nil) - if err != nil { - return nil, err - } - selector := labels.SelectorFromSet(map[string]string{ - kube.RadixComponentLabel: componentName, - kube.RadixJobTypeLabel: kube.RadixJobTypeJobSchedule, - }).Add(*batchNameNotExistsRequirement) - return h.getJobsForLabelSelector(ctx, namespace, selector) -} - -func (h legacyJobHandler) getBatches(ctx context.Context, namespace, componentName string) ([]batchv1.Job, error) { - jobLabelSelector := map[string]string{ - kube.RadixComponentLabel: componentName, - kube.RadixJobTypeLabel: kube.RadixJobTypeBatchSchedule, - } - return h.getJobsForLabelSelector(ctx, namespace, labels.SelectorFromSet(jobLabelSelector)) -} - -func (h legacyJobHandler) getBatchJobs(ctx context.Context, namespace, componentName, batchName string) ([]batchv1.Job, error) { - labelSelector := map[string]string{ - kube.RadixComponentLabel: componentName, - kube.RadixJobTypeLabel: kube.RadixJobTypeJobSchedule, - kube.RadixBatchNameLabel: batchName, - } - return h.getJobsForLabelSelector(ctx, namespace, labels.SelectorFromSet(labelSelector)) -} - -func (h legacyJobHandler) getJobsForLabelSelector(ctx context.Context, namespace string, labelSelector labels.Selector) ([]batchv1.Job, error) { - jobList, err := h.accounts.UserAccount.Client.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: labelSelector.String(), - }) - if err != nil { - return nil, fmt.Errorf("error getting jobs: %w", err) - } - return jobList.Items, err -} - -func (h legacyJobHandler) getJob(ctx context.Context, namespace, componentName, name, jobType string) (*batchv1.Job, error) { - job, err := h.accounts.UserAccount.Client.BatchV1().Jobs(namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - if strings.EqualFold(job.Labels[kube.RadixComponentLabel], componentName) && - strings.EqualFold(job.Labels[kube.RadixJobTypeLabel], jobType) { - return job, nil - } - return nil, jobNotFoundError(name) -} diff --git a/api/environments/legacy_job_status.go b/api/environments/legacy_job_status.go deleted file mode 100644 index 79d269f4..00000000 --- a/api/environments/legacy_job_status.go +++ /dev/null @@ -1,167 +0,0 @@ -package environments - -import ( - "context" - "fmt" - "sort" - "strings" - - jobSchedulerCommonModels "github.com/equinor/radix-job-scheduler/models/common" - jobSchedulerV1Models "github.com/equinor/radix-job-scheduler/models/v1" - "github.com/equinor/radix-operator/pkg/apis/kube" - - "github.com/equinor/radix-common/utils" - v1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -// TODO: Remove this when all legacy jobs are gone - -var imageErrors = map[string]bool{"ImagePullBackOff": true, "ImageInspectError": true, "ErrImagePull": true, - "ErrImageNeverPull": true, "RegistryUnavailable": true, "InvalidImageName": true} - -// GetBatchStatusFromJob Gets job from a k8s jobs for the batch -func GetBatchStatusFromJob(kubeClient kubernetes.Interface, job *v1.Job, jobPods []corev1.Pod) (*jobSchedulerV1Models.BatchStatus, error) { - batchJobStatus := GetJobStatusFromJob(kubeClient, job, jobPods) - batchJobStatus.BatchName = job.GetName() - batchStatus := jobSchedulerV1Models.BatchStatus{ - JobStatus: *batchJobStatus, - } - return &batchStatus, nil -} - -// GetJobStatusFromJob Gets job from a k8s job -func GetJobStatusFromJob(kubeClient kubernetes.Interface, job *v1.Job, jobPods []corev1.Pod) *jobSchedulerV1Models.JobStatus { - jobStatus := jobSchedulerV1Models.JobStatus{ - Name: job.GetName(), - Created: utils.FormatTime(&job.ObjectMeta.CreationTimestamp), - Started: utils.FormatTime(job.Status.StartTime), - Ended: getJobEndTimestamp(job), - } - status := jobSchedulerCommonModels.GetStatusFromJobStatus(job.Status) - - jobStatus.Status = status.String() - jobStatus.JobId = job.ObjectMeta.Labels["radix-job-id"] // Not empty, if JobId exists - jobStatus.BatchName = job.ObjectMeta.Labels[kube.RadixBatchNameLabel] // Not empty, if BatchName exists - if status != jobSchedulerCommonModels.Running { - // if the job is not in state 'Running', we check that job's pod status reason - for _, pod := range jobPods { - if pod.Status.Reason == "DeadlineExceeded" { - // if the pod's status reason is 'DeadlineExceeded', the entire job also gets that status - jobStatus.Status = jobSchedulerCommonModels.DeadlineExceeded.String() - jobStatus.Message = pod.Status.Message - return &jobStatus - } - } - return &jobStatus - } - for _, pod := range jobPods { - if len(pod.Status.ContainerStatuses) > 0 { - cs := pod.Status.ContainerStatuses[0] - if cs.Ready { - continue - } - switch { - case cs.State.Terminated != nil: - // job with one or more 'terminated' containers gets status 'Stopped' - jobStatus.Status = jobSchedulerCommonModels.Stopped.String() - jobStatus.Message = cs.State.Terminated.Message - - return &jobStatus - case cs.State.Waiting != nil: - if _, ok := imageErrors[cs.State.Waiting.Reason]; ok { - // if container waits because of inaccessible image, the job is 'Failed' - jobStatus.Status = jobSchedulerCommonModels.Failed.String() - } else { - // if container waits for any other reason, job is 'Waiting' - jobStatus.Status = jobSchedulerCommonModels.Waiting.String() - } - jobStatus.Started = "" - message := cs.State.Waiting.Message - if len(message) > 0 { - jobStatus.Message = message - return &jobStatus - } - jobStatus.Message = getLastEventMessageForPod(kubeClient, pod) - if len(jobStatus.Message) == 0 { - jobStatus.Message = "Job has not been started. If it takes long time to start, please check an events list for a reason." - } - return &jobStatus - } - continue - } - if len(pod.Status.Conditions) > 0 { - - lastCondition := sortPodStatusConditionsDesc(pod.Status.Conditions)[0] - if lastCondition.Status == corev1.ConditionTrue { - continue - } - jobStatus.Status = jobSchedulerCommonModels.Waiting.String() - jobStatus.Message = fmt.Sprintf("%s %s", lastCondition.Reason, lastCondition.Message) - } - } - return &jobStatus -} - -func getJobEndTimestamp(job *v1.Job) string { - if job.Status.CompletionTime != nil { - // if the k8s job succeeds, we simply return the CompletionTime - return utils.FormatTime(job.Status.CompletionTime) - } - // if a k8s job fails, there is no timestamp for the failure. We set the job's failure time to be - // the timestamp for the job's last status condition. - if job.Status.Conditions != nil { - lastCondition := sortJobStatusConditionsDesc(job.Status.Conditions)[0].LastTransitionTime - return utils.FormatTime(&lastCondition) - } - return "" -} - -func getLastEventMessageForPod(kubeClient kubernetes.Interface, pod corev1.Pod) string { - eventsList, err := kubeClient.CoreV1().Events(pod.Namespace).List(context.Background(), metav1.ListOptions{}) - if err != nil { - return "" - } - events := sortEventsDesc(eventsList.Items) - for _, event := range events { - if event.InvolvedObject.Name == pod.Name { - if strings.Contains(event.Message, "container init was OOM-killed (memory limit too low?)") { - return fmt.Sprintf("Memory limit is probably too low. Error: %s", event.Message) - } - return event.Message - } - } - return "" -} - -func sortEventsDesc(events []corev1.Event) []corev1.Event { - sort.Slice(events, func(i, j int) bool { - if events[i].CreationTimestamp.IsZero() || events[j].CreationTimestamp.IsZero() { - return false - } - return events[j].CreationTimestamp.Before(&events[i].CreationTimestamp) - }) - return events -} - -func sortPodStatusConditionsDesc(podConditions []corev1.PodCondition) []corev1.PodCondition { - sort.Slice(podConditions, func(i, j int) bool { - if podConditions[i].LastTransitionTime.IsZero() || podConditions[j].LastTransitionTime.IsZero() { - return false - } - return podConditions[j].LastTransitionTime.Before(&podConditions[i].LastTransitionTime) - }) - return podConditions -} - -func sortJobStatusConditionsDesc(jobConditions []v1.JobCondition) []v1.JobCondition { - sort.Slice(jobConditions, func(i, j int) bool { - if jobConditions[i].LastTransitionTime.IsZero() || jobConditions[j].LastTransitionTime.IsZero() { - return false - } - return jobConditions[j].LastTransitionTime.Before(&jobConditions[i].LastTransitionTime) - }) - return jobConditions -}