From 018fb8b6ce98e887f5c96c301cdb13fd31e9f1c4 Mon Sep 17 00:00:00 2001 From: Prasad Saraf Date: Thu, 25 Aug 2022 17:55:22 +0530 Subject: [PATCH 1/2] Cleanup: Use ResourceFlavor.metadata.labels Change-Id: I6d4cbe5f48a11ce252bd62aff1688adbe1b273e6 --- apis/kueue/v1alpha2/resourceflavor_types.go | 14 ++++------ apis/kueue/v1alpha2/zz_generated.deepcopy.go | 7 ----- apis/kueue/webhooks/resourceflavor_webhook.go | 4 +-- .../webhooks/resourceflavor_webhook_test.go | 20 +------------ .../bases/kueue.x-k8s.io_resourceflavors.yaml | 14 +++------- docs/concepts/cluster_queue.md | 6 ++-- docs/tasks/administer_cluster_quotas.md | 8 +++--- pkg/cache/cache_test.go | 18 ++++++++---- pkg/cache/snapshot_test.go | 26 ++++++++++------- pkg/scheduler/scheduler_test.go | 24 ++++++++++------ pkg/util/testing/wrappers.go | 4 +-- .../webhook/v1alpha2/resourceflavor_test.go | 28 ++++++++++++------- 12 files changed, 82 insertions(+), 91 deletions(-) diff --git a/apis/kueue/v1alpha2/resourceflavor_types.go b/apis/kueue/v1alpha2/resourceflavor_types.go index 0ccbf2a774..1029b8ca9b 100644 --- a/apis/kueue/v1alpha2/resourceflavor_types.go +++ b/apis/kueue/v1alpha2/resourceflavor_types.go @@ -24,19 +24,15 @@ import ( //+kubebuilder:object:root=true //+kubebuilder:resource:scope=Cluster -// ResourceFlavor is the Schema for the resourceflavors API +// ResourceFlavor is the Schema for the resourceflavors API. +// +// .metadata.labels associated with this flavor are matched against or +// converted to node affinity constraints on the workload’s pods. +// .metadata.labels can be up to 8 elements. type ResourceFlavor struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - // labels associated with this flavor. They are matched against or - // converted to node affinity constraints on the workload’s pods. - // For example, cloud.provider.com/accelerator: nvidia-tesla-k80. - // More info: http://kubernetes.io/docs/user-guide/labels - // - // labels can be up to 8 elements. - Labels map[string]string `json:"labels,omitempty"` - // taints associated with this flavor that workloads must explicitly // “tolerate” to be able to use this flavor. // For example, cloud.provider.com/preemptible="true":NoSchedule diff --git a/apis/kueue/v1alpha2/zz_generated.deepcopy.go b/apis/kueue/v1alpha2/zz_generated.deepcopy.go index c464683115..1e9200f140 100644 --- a/apis/kueue/v1alpha2/zz_generated.deepcopy.go +++ b/apis/kueue/v1alpha2/zz_generated.deepcopy.go @@ -358,13 +358,6 @@ func (in *ResourceFlavor) DeepCopyInto(out *ResourceFlavor) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } if in.Taints != nil { in, out := &in.Taints, &out.Taints *out = make([]corev1.Taint, len(*in)) diff --git a/apis/kueue/webhooks/resourceflavor_webhook.go b/apis/kueue/webhooks/resourceflavor_webhook.go index 9fbd4130ba..23e1f8c7ef 100644 --- a/apis/kueue/webhooks/resourceflavor_webhook.go +++ b/apis/kueue/webhooks/resourceflavor_webhook.go @@ -88,11 +88,9 @@ func (w *ResourceFlavorWebhook) ValidateDelete(ctx context.Context, obj runtime. func ValidateResourceFlavor(rf *kueue.ResourceFlavor) field.ErrorList { var allErrs field.ErrorList - labelsPath := field.NewPath("labels") if len(rf.Labels) > 8 { - allErrs = append(allErrs, field.TooMany(labelsPath, len(rf.Labels), 8)) + allErrs = append(allErrs, field.TooMany(field.NewPath("metadata", "labels"), len(rf.Labels), 8)) } - allErrs = append(allErrs, metavalidation.ValidateLabels(rf.Labels, labelsPath)...) taintsPath := field.NewPath("taints") if len(rf.Taints) > 8 { diff --git a/apis/kueue/webhooks/resourceflavor_webhook_test.go b/apis/kueue/webhooks/resourceflavor_webhook_test.go index 29ffc93a88..a7767b1295 100644 --- a/apis/kueue/webhooks/resourceflavor_webhook_test.go +++ b/apis/kueue/webhooks/resourceflavor_webhook_test.go @@ -50,24 +50,6 @@ func TestValidateResourceFlavor(t *testing.T) { Effect: corev1.TaintEffectNoSchedule, }).Obj(), }, - { - name: "invalid label name", - rf: utiltesting.MakeResourceFlavor("resource-flavor").MultiLabels(map[string]string{ - "foo@bar": "", - }).Obj(), - wantErr: field.ErrorList{ - field.Invalid(field.NewPath("labels"), nil, ""), - }, - }, - { - name: "invalid label value", - rf: utiltesting.MakeResourceFlavor("resource-flavor").MultiLabels(map[string]string{ - "foo": "@abcdefg", - }).Obj(), - wantErr: field.ErrorList{ - field.Invalid(field.NewPath("labels"), nil, ""), - }, - }, { // Taint validation is not exhaustively tested, because the code was copied from upstream k8s. name: "invalid taint", @@ -88,7 +70,7 @@ func TestValidateResourceFlavor(t *testing.T) { return m }()).Obj(), wantErr: field.ErrorList{ - field.TooMany(field.NewPath("labels"), 9, 8), + field.TooMany(field.NewPath("metadata", "labels"), 9, 8), }, }, { diff --git a/config/components/crd/bases/kueue.x-k8s.io_resourceflavors.yaml b/config/components/crd/bases/kueue.x-k8s.io_resourceflavors.yaml index 7aa4d1a040..6acda11414 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_resourceflavors.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_resourceflavors.yaml @@ -18,7 +18,10 @@ spec: - name: v1alpha2 schema: openAPIV3Schema: - description: ResourceFlavor is the Schema for the resourceflavors API + description: "ResourceFlavor is the Schema for the resourceflavors API. \n + .metadata.labels associated with this flavor are matched against or converted + to node affinity constraints on the workload’s pods. .metadata.labels can + be up to 8 elements." properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -30,15 +33,6 @@ spec: object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string - labels: - additionalProperties: - type: string - description: "labels associated with this flavor. They are matched against - or converted to node affinity constraints on the workload’s pods. For - example, cloud.provider.com/accelerator: nvidia-tesla-k80. More info: - http://kubernetes.io/docs/user-guide/labels \n labels can be up to 8 - elements." - type: object metadata: type: object taints: diff --git a/docs/concepts/cluster_queue.md b/docs/concepts/cluster_queue.md index 3c612f9cee..d8a5a1af5f 100644 --- a/docs/concepts/cluster_queue.md +++ b/docs/concepts/cluster_queue.md @@ -163,8 +163,8 @@ apiVersion: kueue.x-k8s.io/v1alpha1 kind: ResourceFlavor metadata: name: spot -labels: - instance-type: spot + labels: + instance-type: spot taints: - effect: NoSchedule key: spot @@ -177,7 +177,7 @@ ClusterQueue in the `.spec.resources[*].flavors[*].name` field. ### ResourceFlavor labels To associate a ResourceFlavor with a subset of nodes of you cluster, you can -configure the `.labels` field with matching node labels that uniquely identify +configure the `.metadata.labels` field with matching node labels that uniquely identify the nodes. If you are using [cluster autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) (or equivalent controllers), make sure it is configured to add those labels when adding new nodes. diff --git a/docs/tasks/administer_cluster_quotas.md b/docs/tasks/administer_cluster_quotas.md index 43a8225b20..d2da5d0339 100644 --- a/docs/tasks/administer_cluster_quotas.md +++ b/docs/tasks/administer_cluster_quotas.md @@ -143,8 +143,8 @@ apiVersion: kueue.x-k8s.io/v1alpha1 kind: ResourceFlavor metadata: name: x86 -labels: - cpu-arch: x86 + labels: + cpu-arch: x86 ``` ```yaml @@ -153,8 +153,8 @@ apiVersion: kueue.x-k8s.io/v1alpha1 kind: ResourceFlavor metadata: name: arm -labels: - cpu-arch: arm + labels: + cpu-arch: arm ``` To create the ResourceFlavors, run the following command: diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 962edc64d9..e3c10fed16 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -103,8 +103,10 @@ func TestCacheClusterQueueOperations(t *testing.T) { } setup := func(cache *Cache) { cache.AddOrUpdateResourceFlavor(&kueue.ResourceFlavor{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Labels: map[string]string{"cpuType": "default"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Labels: map[string]string{"cpuType": "default"}, + }, }) for _, c := range initialClusterQueues { if err := cache.AddClusterQueue(context.Background(), &c); err != nil { @@ -189,8 +191,10 @@ func TestCacheClusterQueueOperations(t *testing.T) { } } cache.AddOrUpdateResourceFlavor(&kueue.ResourceFlavor{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Labels: map[string]string{"cpuType": "default"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Labels: map[string]string{"cpuType": "default"}, + }, }) }, wantClusterQueues: map[string]*ClusterQueue{ @@ -302,8 +306,10 @@ func TestCacheClusterQueueOperations(t *testing.T) { } } cache.AddOrUpdateResourceFlavor(&kueue.ResourceFlavor{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Labels: map[string]string{"cpuType": "default", "region": "central"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Labels: map[string]string{"cpuType": "default", "region": "central"}, + }, }) }, wantClusterQueues: map[string]*ClusterQueue{ diff --git a/pkg/cache/snapshot_test.go b/pkg/cache/snapshot_test.go index 72971f8ec0..f140bcf662 100644 --- a/pkg/cache/snapshot_test.go +++ b/pkg/cache/snapshot_test.go @@ -152,16 +152,19 @@ func TestSnapshot(t *testing.T) { } flavors := []kueue.ResourceFlavor{ { - ObjectMeta: metav1.ObjectMeta{Name: "demand"}, - Labels: map[string]string{"foo": "bar", "instance": "demand"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "demand", + Labels: map[string]string{"foo": "bar", "instance": "demand"}, + }, }, { - ObjectMeta: metav1.ObjectMeta{Name: "spot"}, - Labels: map[string]string{"baz": "bar", "instance": "spot"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "spot", + Labels: map[string]string{"baz": "bar", "instance": "spot"}, + }, }, { ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Labels: nil, }, } @@ -381,15 +384,18 @@ func TestSnapshot(t *testing.T) { ResourceFlavors: map[string]*kueue.ResourceFlavor{ "default": { ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Labels: nil, }, "demand": { - ObjectMeta: metav1.ObjectMeta{Name: "demand"}, - Labels: map[string]string{"foo": "bar", "instance": "demand"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "demand", + Labels: map[string]string{"foo": "bar", "instance": "demand"}, + }, }, "spot": { - ObjectMeta: metav1.ObjectMeta{Name: "spot"}, - Labels: map[string]string{"baz": "bar", "instance": "spot"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "spot", + Labels: map[string]string{"baz": "bar", "instance": "spot"}, + }, }, }, InactiveClusterQueueSets: sets.String{"flavor-nonexistent-cq": {}}, diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index a3d4221058..8672d07b1d 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -885,20 +885,28 @@ func TestEntryAssignFlavors(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "default"}, }, "one": { - ObjectMeta: metav1.ObjectMeta{Name: "one"}, - Labels: map[string]string{"type": "one"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "one", + Labels: map[string]string{"type": "one"}, + }, }, "two": { - ObjectMeta: metav1.ObjectMeta{Name: "two"}, - Labels: map[string]string{"type": "two"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "two", + Labels: map[string]string{"type": "two"}, + }, }, "b_one": { - ObjectMeta: metav1.ObjectMeta{Name: "b_one"}, - Labels: map[string]string{"b_type": "one"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "b_one", + Labels: map[string]string{"b_type": "one"}, + }, }, "b_two": { - ObjectMeta: metav1.ObjectMeta{Name: "b_two"}, - Labels: map[string]string{"b_type": "two"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "b_two", + Labels: map[string]string{"b_type": "two"}, + }, }, "tainted": { ObjectMeta: metav1.ObjectMeta{Name: "tainted"}, diff --git a/pkg/util/testing/wrappers.go b/pkg/util/testing/wrappers.go index 06cbc85651..3ea3058975 100644 --- a/pkg/util/testing/wrappers.go +++ b/pkg/util/testing/wrappers.go @@ -385,9 +385,9 @@ type ResourceFlavorWrapper struct{ kueue.ResourceFlavor } func MakeResourceFlavor(name string) *ResourceFlavorWrapper { return &ResourceFlavorWrapper{kueue.ResourceFlavor{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: name, + Labels: map[string]string{}, }, - Labels: map[string]string{}, }} } diff --git a/test/integration/webhook/v1alpha2/resourceflavor_test.go b/test/integration/webhook/v1alpha2/resourceflavor_test.go index fb3b48b511..98e9d04a68 100644 --- a/test/integration/webhook/v1alpha2/resourceflavor_test.go +++ b/test/integration/webhook/v1alpha2/resourceflavor_test.go @@ -62,37 +62,45 @@ var _ = ginkgo.Describe("ResourceFlavor Webhook", func() { }) }) - ginkgo.When("Creating a ResourceFlavor with invalid labels", func() { + ginkgo.When("Creating a ResourceFlavor with invalid taints", func() { ginkgo.It("Should fail to create", func() { ginkgo.By("Creating a new resourceFlavor") - resourceFlavor := testing.MakeResourceFlavor("resource-flavor").Label("foo", "@abcd").Obj() + resourceFlavor := testing.MakeResourceFlavor("resource-flavor").Taint(corev1.Taint{ + Key: "@foo", + Value: "bar", + Effect: corev1.TaintEffectNoSchedule, + }).Obj() err := k8sClient.Create(ctx, resourceFlavor) - gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(errors.IsForbidden(err)).Should(gomega.BeTrue(), "error: %v", err) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(errors.IsForbidden(err)).To(gomega.BeTrue(), "error: %v", err) }) }) - ginkgo.When("Updating a ResourceFlavor with invalid labels", func() { + ginkgo.When("Updating a ResourceFlavor with invalid taints", func() { ginkgo.It("Should fail to update", func() { ginkgo.By("Creating a new resourceFlavor") resourceFlavor := testing.MakeResourceFlavor("resource-flavor").Obj() gomega.Expect(k8sClient.Create(ctx, resourceFlavor)).Should(gomega.Succeed()) defer func() { var rf kueue.ResourceFlavor - gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(resourceFlavor), &rf)).Should(gomega.Succeed()) + gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(resourceFlavor), &rf)).To(gomega.Succeed()) controllerutil.RemoveFinalizer(&rf, kueue.ResourceInUseFinalizerName) gomega.Expect(k8sClient.Update(ctx, &rf)).Should(gomega.Succeed()) framework.ExpectResourceFlavorToBeDeleted(ctx, k8sClient, resourceFlavor, true) }() var created kueue.ResourceFlavor - gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(resourceFlavor), &created)).Should(gomega.Succeed()) - created.Labels = map[string]string{"foo": "@abcd"} + gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(resourceFlavor), &created)).To(gomega.Succeed()) + created.Taints = []corev1.Taint{{ + Key: "foo", + Value: "bar", + Effect: "Invalid", + }} ginkgo.By("Updating the resourceFlavor with invalid labels") err := k8sClient.Update(ctx, &created) - gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(errors.IsForbidden(err)).Should(gomega.BeTrue(), "error: %v", err) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(errors.IsForbidden(err)).To(gomega.BeTrue(), "error: %v", err) }) }) }) From 801697176f94eea409fdff5e89c69b9e2a53456e Mon Sep 17 00:00:00 2001 From: Aldo Culquicondor <1299064+alculquicondor@users.noreply.github.com> Date: Thu, 25 Aug 2022 15:10:08 -0400 Subject: [PATCH 2/2] Recover updates to docs and release notes (#356) * Recover updates to docs and release notes * Add more features to release notes Change-Id: I1783a3e890da9a0599b83452e77d956e58d83ec6 * Better wording Change-Id: Idb8737e5ba5f55863438d601743a15e5967f6ea1 --- CHANGELOG/CHANGELOG-0.2.md | 30 +++++++++------ README.md | 17 ++++----- docs/concepts/cluster_queue.md | 69 ++++++++++++++++++++-------------- docs/concepts/local_queue.md | 4 +- docs/setup/install.md | 3 +- 5 files changed, 71 insertions(+), 52 deletions(-) diff --git a/CHANGELOG/CHANGELOG-0.2.md b/CHANGELOG/CHANGELOG-0.2.md index ad7ad2b1f3..7b98690d29 100644 --- a/CHANGELOG/CHANGELOG-0.2.md +++ b/CHANGELOG/CHANGELOG-0.2.md @@ -3,24 +3,32 @@ Changes since `v0.1.0`: ### Features -- Bumped the API version from v1alpha1 to v1alpha2. v1alpha1 is no longer supported and Queue is now named LocalQueue. -- Add webhooks to validate and add defaults to all kueue APIs. + +- Upgrade the API version from v1alpha1 to v1alpha2. v1alpha1 is no longer supported. + v1alpha2 includes the following changes: + - Rename Queue to LocalQueue. + - Remove ResourceFlavor.labels. Use ResourceFlavor.metadata.labels instead. +- Add webhooks to validate and to add defaults to all kueue APIs. +- Add internal cert manager to serve webhooks with TLS. +- Use finalizers to prevent ClusterQueues and ResourceFlavors in use from being + deleted prematurely. - Support [codependent resources](/docs/concepts/cluster_queue.md#codepedent-resources) by assigning the same flavor to codependent resources in a pod set. - Support [pod overhead](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-overhead/) in Workload pod sets. -- Default requests to limits if requests are not set in a Workload pod set, to - match internal defaulting for k8s Pods. -- Added [prometheus metrics](/docs/reference/metrics.md) to monitor health of +- Set requests to limits if requests are not set in a Workload pod set, + matching [internal defaulting for k8s Pods](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources). +- Add [prometheus metrics](/docs/reference/metrics.md) to monitor health of the system and the status of ClusterQueues. +- Use Server Side Apply for Workload admission to reduce API conflicts. ### Bug fixes -- Prevent Workloads that don't match the ClusterQueue's namespaceSelector from - blocking other Workloads in a StrictFIFO ClusterQueue. -- Fixed number of pending workloads in a BestEffortFIFO ClusterQueue. -- Fixed bug in a BestEffortFIFO ClusterQueue where a workload might not be +- Fix bug that caused Workloads that don't match the ClusterQueue's + namespaceSelector to block other Workloads in StrictFIFO ClusterQueues. +- Fix the number of pending workloads in BestEffortFIFO ClusterQueues status. +- Fix a bug in BestEffortFIFO ClusterQueues where a workload might not be retried after a transient error. -- Fixed requeuing an out-of-date workload when failed to admit it. -- Fixed bug in a BestEffortFIFO ClusterQueue where unadmissible workloads +- Fix requeuing an out-of-date workload when failed to admit it. +- Fix a bug in BestEffortFIFO ClusterQueues where inadmissible workloads were not removed from the ClusterQueue when removing the corresponding Queue. diff --git a/README.md b/README.md index db60a047df..9e38246e88 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,15 @@ created) and when it should stop (as in active pods should be deleted). ## Why use Kueue Kueue is a lean controller that you can install on top of a vanilla Kubernetes -cluster without replacing any components. It is compatible with cloud -environments where: -- Nodes and other compute resources can be scaled up and down. +cluster. Kueue does not replace any existing Kubernetes components. Kueue is +compatible with cloud environments where: +- Compute resources are elastic and can be scaled up and down. - Compute resources are heterogeneous (in architecture, availability, price, etc.). Kueue APIs allow you to express: - Quotas and policies for fair sharing among tenants. - Resource fungibility: if a [resource flavor](docs/concepts/cluster_queue.md#resourceflavor-object) - is fully utilized, run the [job](docs/concepts/workload.md) using a different - flavor. + is fully utilized, Kueue can admit the job using a different flavor. The main design principle for Kueue is to avoid duplicating mature functionality in [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) @@ -62,11 +61,11 @@ Learn more about: -Learn more about the architecture of Kueue in the design docs: +Learn more about the architecture of Kueue with the following design docs: -- [bit.ly/kueue-apis](https://bit.ly/kueue-apis) (please join the [mailing list](https://groups.google.com/a/kubernetes.io/g/wg-batch) -to get access) discusses the API proposal and a high-level description of how it -operates. +- [bit.ly/kueue-apis](https://bit.ly/kueue-apis) discusses the API proposal and a high + level description of how Kueue operates. Join the [mailing list](https://groups.google.com/a/kubernetes.io/g/wg-batch) +to get document access. - [bit.ly/kueue-controller-design](https://bit.ly/kueue-controller-design) presents the detailed design of the controller. diff --git a/docs/concepts/cluster_queue.md b/docs/concepts/cluster_queue.md index d8a5a1af5f..bf25feeb80 100644 --- a/docs/concepts/cluster_queue.md +++ b/docs/concepts/cluster_queue.md @@ -1,9 +1,9 @@ # Cluster Queue A ClusterQueue is a cluster-scoped object that governs a pool of resources -such as CPU, memory and hardware accelerators. A `ClusterQueue` defines: -- The [resource _flavors_](#resourceflavor-object) that it manages, with usage - limits and order of consumption. +such as CPU, memory, and hardware accelerators. A ClusterQueue defines: +- The [resource _flavors_](#resourceflavor-object) that the ClusterQueue manages, + with usage limits and order of consumption. - Fair sharing rules across the tenants of the cluster. Only [cluster administrators](/docs/tasks#batch-administrator) should create `ClusterQueue` objects. @@ -39,29 +39,29 @@ You can specify the quota as a [quantity](https://kubernetes.io/docs/reference/k ## Resources In a ClusterQueue, you can define quotas for multiple [compute resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-types) -(cpu, memory, GPUs, etc.). +(CPU, memory, GPUs, etc.). -For each resource, you can define quotas for multiple _flavors_. A -flavor represents different variations of a resource. The variations can be -defined in a [ResourceFlavor object](#resourceflavor-object). +For each resource, you can define quotas for multiple _flavors_. +Flavors represent different variations of a resource (for example, different GPU +models). A flavor is defined using a [ResourceFlavor object](#resourceflavor-object). -In a process called [admission](.#admission), Kueue assigns -[Workload pod sets](workload.md#pod-sets) a flavor for each resource it requests. +In a process called [admission](.#admission), Kueue assigns to the +[Workload pod sets](workload.md#pod-sets) a flavor for each resource the pod set +requests. Kueue assigns the first flavor in the ClusterQueue's `.spec.resources[*].flavors` list that has enough unused `min` quota in the ClusterQueue or the ClusterQueue's [cohort](#cohort). ### Codepedent resources -It is possible that multiple resources are tied to the same flavors. This is -typical for `cpu` and `memory`, where the flavors are generally tied to a -machine family or availability guarantees. +It is possible that multiple resources in a ClusterQueue have the same flavors. +This is typical for `cpu` and `memory`, where the flavors are generally tied to +a machine family or VM availability policies. When two or more resources in a +ClusterQueue match their flavors, they are said to be codependent resources. -If this is the case, the resources in the ClusterQueue must list the same -flavors in the same order. When two or more resources match their flavors, -they are said to be codependent. During admission, for each pod set in a -Workload, Kueue assigns the same flavor to the codependent resources that the -pod set requests. +To manage codependent resources, you should list the flavors in the ClusterQueue +resources in the same order. During admission, for each pod set in a Workload, +Kueue assigns the same flavor to the codependent resources that the pod set requests. An example of a ClusterQueue with codependent resources looks like the following: @@ -150,8 +150,8 @@ Resources in a cluster are typically not homogeneous. Resources could differ in: - architecture (ex: x86 vs ARM CPUs) - brands and models (ex: Radeon 7000 vs Nvidia A100 vs T4 GPUs) -A ResourceFlavor is an object that represents these variations and allows you -to associate them with node labels and taints. +A ResourceFlavor is an object that represents these resource variations and +allows you to associate them with node labels and taints. **Note**: If your cluster is homogeneous, you can use an [empty ResourceFlavor](#empty-resourceflavor) instead of adding labels to custom ResourceFlavors. @@ -197,8 +197,8 @@ steps: For example, for a [batch/v1.Job](https://kubernetes.io/docs/concepts/workloads/controllers/job/), Kueue adds the labels to the `.spec.template.spec.nodeSelector` field. This - guarantees that the workload Pods run on the nodes associated to the flavor - that Kueue decided that the workload should use. + guarantees that the Workload's Pods can only be scheduled on the nodes + targeted by the flavor that Kueue assigned to the Workload. ### ResourceFlavor taints @@ -208,7 +208,7 @@ with taints. Taints on the ResourceFlavor work similarly to [node taints](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). For Kueue to admit a workload to use the ResourceFlavor, the PodSpecs in the workload should have a toleration for it. As opposed to the behavior for -[ResourceFlavor labels](#resourceflavor-labels), Kueue will not add tolerations +[ResourceFlavor labels](#resourceflavor-labels), Kueue does not add tolerations for the flavor taints. ### Empty ResourceFlavor @@ -238,16 +238,27 @@ ClusterQueue. ### Flavors and borrowing semantics -When borrowing, Kueue satisfies the following admission semantics: +When a ClusterQueue is part of a cohort, Kueue satisfies the following admission +semantics: - When assigning flavors, Kueue goes through the list of flavors in the ClusterQueue's `.spec.resources[*].flavors`. For each flavor, Kueue attempts - to fit a Workload's pod set using the `min` quota of the ClusterQueue or the - unused `min` quota of other ClusterQueues in the cohort, up to the `max` quota - of the ClusterQueue. If the workload doesn't fit, Kueue proceeds evaluating the next - flavor in the list. -- A ClusterQueue can only borrow quota of flavors it defines and it can only - borrow quota for one flavor. + to fit a Workload's pod set according to the quota defined in the + ClusterQueue for the flavor and the unused quota in the cohort. + If the workload doesn't fit, Kueue evaluates the next flavor in the list. +- A Workload's pod set resource fits in a flavor defined for a ClusterQueue + resource if the sum of requests for the resource: + 1. Is less than or equal to the unused `.quota.min` for the flavor in the + ClusterQueue; or + 2. Is less than or equal to the sum of unused `.quota.min` for the flavor in + the ClusterQueues in the cohort, and + 3. Is less than or equal to the unused `.quota.max` for the flavor in the + ClusterQueue. + In Kueue, when (2) and (3) are satisfied, but not (1), this is called + _borrowing quota_. +- A ClusterQueue can only borrow quota for flavors that the ClusterQueue defines. +- For each pod set resource in a Workload, a ClusterQueue can only borrow quota + for one flavor. ### Borrowing example diff --git a/docs/concepts/local_queue.md b/docs/concepts/local_queue.md index de984334b3..b55c9ad724 100644 --- a/docs/concepts/local_queue.md +++ b/docs/concepts/local_queue.md @@ -4,9 +4,9 @@ A `LocalQueue` is a namespaced object that groups closely related workloads belonging to a single tenant. A `LocalQueue` points to one [`ClusterQueue`](cluster_queue.md) from which resources are allocated to run its workloads. -Users submit jobs to a `LocalQueue`, instead of directly to a `ClusterQueue`. +Users submit jobs to a `LocalQueue`, instead of to a `ClusterQueue` directly. Tenants can discover which queues they can submit jobs to by listing the -local queues in their namespace. The command looks similar to the following: +local queues in their namespace. The command is similar to the following: ```sh kubectl get -n my-namespace localqueues diff --git a/docs/setup/install.md b/docs/setup/install.md index 75c48de8ac..8d5df0ebc2 100644 --- a/docs/setup/install.md +++ b/docs/setup/install.md @@ -50,7 +50,8 @@ kubectl delete -f https://github.com/kubernetes-sigs/kueue/releases/download/$VE ### Upgrading from 0.1 to 0.2 -Upgrading from `0.1.x` to `0.2.y` is not supported due to breaking API changes. +Upgrading from `0.1.x` to `0.2.y` is not supported because of breaking API +changes. To install Kueue `0.2.y`, [uninstall](#uninstall) the older version first. ## Install a custom-configured released version