From e671b4f12bf4e36f55c6e8cbaede39ea97d8716a Mon Sep 17 00:00:00 2001 From: Dan Norris Date: Thu, 28 Mar 2024 09:11:12 -0400 Subject: [PATCH] feat: enable additional scheduling options for wasmCloud host pods This refactors the WasmCloudHostConfig CRD so that it has a single field (`schedulingOptions`) for configuring how the underlying Pods are scheduled in Kubernetes. This includes: * Relocating the `daemonset` option to this new field * Relocating the `resources` option to this new field * Adding a new `pod_template_additions` field that allows you to set any valid option in a `PodSpec` Doing so allows cluster operators to do things like set node affinity and node selector rules, along with any other valid PodSpec option. The only thing that cannot be done is adding additional containers to the pod, since that is all handled by the controller. We could look at exposing that option if users want to be able to add additional sidecars. Signed-off-by: Dan Norris --- Cargo.lock | 2 +- Cargo.toml | 2 +- Dockerfile.local | 2 +- .../src/v1alpha1/wasmcloud_host_config.rs | 40 ++++++-- sample.yaml | 22 ++++- src/controller.rs | 99 +++++++++++++------ 6 files changed, 125 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0347ff2..4595c33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4302,7 +4302,7 @@ dependencies = [ [[package]] name = "wasmcloud-operator" -version = "0.1.1" +version = "0.2.0" dependencies = [ "anyhow", "async-nats", diff --git a/Cargo.toml b/Cargo.toml index 67927f1..450babe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wasmcloud-operator" -version = "0.1.1" +version = "0.2.0" edition = "2021" [[bin]] diff --git a/Dockerfile.local b/Dockerfile.local index 44263d5..f386202 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM rust:1.75-bookworm as builder +FROM rust:1.77-bookworm as builder WORKDIR /app COPY . . diff --git a/crates/types/src/v1alpha1/wasmcloud_host_config.rs b/crates/types/src/v1alpha1/wasmcloud_host_config.rs index bd0c4f0..e1157b8 100644 --- a/crates/types/src/v1alpha1/wasmcloud_host_config.rs +++ b/crates/types/src/v1alpha1/wasmcloud_host_config.rs @@ -1,8 +1,8 @@ -use k8s_openapi::api::core::v1::ResourceRequirements; +use k8s_openapi::api::core::v1::{PodSpec, ResourceRequirements}; use kube::CustomResource; -use schemars::JsonSchema; +use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] #[cfg_attr(test, derive(Default))] @@ -36,10 +36,6 @@ pub struct WasmCloudHostConfigSpec { pub enable_structured_logging: Option, /// Name of a secret containing the registry credentials pub registry_credentials_secret: Option, - /// Kubernetes resources to allocate for the host. See - /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for valid - /// values to use here. - pub resources: Option, /// The control topic prefix to use for the host. pub control_topic_prefix: Option, /// The leaf node domain to use for the NATS sidecar. Defaults to "leaf". @@ -57,9 +53,39 @@ pub struct WasmCloudHostConfigSpec { /// The log level to use for the host. Defaults to "INFO". #[serde(default = "default_log_level")] pub log_level: String, + /// Kubernetes scheduling options for the wasmCloud host. + pub scheduling_options: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct KubernetesSchedulingOptions { /// Run hosts as a DaemonSet instead of a Deployment. #[serde(default)] pub daemonset: bool, + /// Kubernetes resources to allocate for the host. See + /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for valid + /// values to use here. + pub resources: Option, + #[schemars(schema_with = "pod_schema")] + /// Any other pod template spec options to set for the underlying wasmCloud host pods. + pub pod_template_additions: Option, +} + +/// This is a workaround for the fact that we can't override the PodSpec schema to make containers +/// an optional field. It generates the OpenAPI schema for the PodSpec type the same way that +/// kube.rs does while dropping any required fields. +fn pod_schema(_gen: &mut SchemaGenerator) -> Schema { + let gen = schemars::gen::SchemaSettings::openapi3() + .with(|s| { + s.inline_subschemas = true; + s.meta_schema = None; + }) + .with_visitor(kube::core::schema::StructuralSchemaRewriter) + .into_generator(); + let mut val = gen.into_root_schema_for::(); + // Drop `containers` as a required field, along with any others. + val.schema.object.as_mut().unwrap().required = BTreeSet::new(); + val.schema.into() } fn default_host_replicas() -> u32 { diff --git a/sample.yaml b/sample.yaml index e3c038a..8f4e879 100644 --- a/sample.yaml +++ b/sample.yaml @@ -18,5 +18,23 @@ spec: secretName: cluster-secrets logLevel: INFO natsAddress: nats://nats-cluster.default.svc.cluster.local - # Enable the following to run the wasmCloud hosts as a DaemonSet - #daemonset: true + # Additional options to control how the underlying wasmCloud hosts are scheduled in Kubernetes. + # This includes setting resource requirements for the nats and wasmCloud host + # containers along with any additional pot template settings. + #schedulingOptions: + # Enable the following to run the wasmCloud hosts as a DaemonSet + #daemonset: true + # Set the resource requirements for the nats and wasmCloud host containers. + #resources: + # nats: + # requests: + # cpu: 100m + # wasmCloudHost: + # requests: + # cpu: 100m + # Any additional pod template settings to apply to the wasmCloud host pods. + # See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#podspec-v1-core for all valid options. + # Note that you *cannot* set the `containers` field here as it is managed by the controller. + #pod_template_additions: + # nodeSelector: + # kubernetes.io/os: linux diff --git a/src/controller.rs b/src/controller.rs index c11a005..a83267d 100644 --- a/src/controller.rs +++ b/src/controller.rs @@ -315,9 +315,11 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc) -> PodTemplate let mut nats_resources: Option = None; let mut wasmcloud_resources: Option = None; - if let Some(resources) = &config.spec.resources { - nats_resources = resources.nats.clone(); - wasmcloud_resources = resources.wasmcloud.clone(); + if let Some(scheduling_options) = &config.spec.scheduling_options { + if let Some(resources) = &scheduling_options.resources { + nats_resources = resources.nats.clone(); + wasmcloud_resources = resources.wasmcloud.clone(); + } } let containers = vec![ @@ -371,14 +373,34 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc) -> PodTemplate ..Default::default() }, ]; - PodTemplateSpec { + + let mut volumes = vec![ + Volume { + name: "nats-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: Some(config.name_any()), + ..Default::default() + }), + ..Default::default() + }, + Volume { + name: "nats-creds".to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(config.spec.secret_name.clone()), + ..Default::default() + }), + ..Default::default() + }, + ]; + let service_account = config.name_any(); + let mut template = PodTemplateSpec { metadata: Some(ObjectMeta { labels: Some(labels), ..Default::default() }), spec: Some(PodSpec { service_account: Some(config.name_any()), - containers, + containers: containers.clone(), volumes: Some(vec![ Volume { name: "nats-config".to_string(), @@ -399,7 +421,21 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc) -> PodTemplate ]), ..Default::default() }), - } + }; + + if let Some(scheduling_options) = &config.spec.scheduling_options { + if let Some(pod_overrides) = &scheduling_options.pod_template_additions { + let mut overrides = pod_overrides.clone(); + overrides.service_account_name = Some(service_account); + overrides.containers = containers.clone(); + if let Some(vols) = overrides.volumes { + volumes.extend(vols); + } + overrides.volumes = Some(volumes); + template.spec = Some(overrides); + } + }; + template } fn deployment_spec(config: &WasmCloudHostConfig, ctx: Arc) -> DeploymentSpec { @@ -504,31 +540,34 @@ async fn configure_hosts(config: &WasmCloudHostConfig, ctx: Arc) -> Res ]; } - if config.spec.daemonset { - let mut spec = daemonset_spec(config, ctx.clone()); - spec.template.spec.as_mut().unwrap().containers[1] - .env - .as_mut() - .unwrap() - .append(&mut env_vars); - let ds = DaemonSet { - metadata: ObjectMeta { - name: Some(config.name_any()), - namespace: Some(config.namespace().unwrap()), - owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]), + if let Some(scheduling_options) = &config.spec.scheduling_options { + if scheduling_options.daemonset { + let mut spec = daemonset_spec(config, ctx.clone()); + spec.template.spec.as_mut().unwrap().containers[1] + .env + .as_mut() + .unwrap() + .append(&mut env_vars); + let ds = DaemonSet { + metadata: ObjectMeta { + name: Some(config.name_any()), + namespace: Some(config.namespace().unwrap()), + owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]), + ..Default::default() + }, + spec: Some(spec), ..Default::default() - }, - spec: Some(spec), - ..Default::default() - }; - - let api = Api::::namespaced(ctx.client.clone(), &config.namespace().unwrap()); - api.patch( - &config.name_any(), - &PatchParams::apply(CLUSTER_CONFIG_FINALIZER), - &Patch::Apply(ds), - ) - .await?; + }; + + let api = + Api::::namespaced(ctx.client.clone(), &config.namespace().unwrap()); + api.patch( + &config.name_any(), + &PatchParams::apply(CLUSTER_CONFIG_FINALIZER), + &Patch::Apply(ds), + ) + .await?; + } } else { let mut spec = deployment_spec(config, ctx.clone()); spec.template.spec.as_mut().unwrap().containers[1]