Merge pull request #9 from wasmCloud/node_affinity

feat: enable additional scheduling options for wasmCloud host pods

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "wasmcloud-operator"
-version = "0.1.1"
+version = "0.2.0"
 edition = "2021"
 
 [[bin]]

Dockerfile.local

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM rust:1.75-bookworm as builder
+FROM rust:1.77-bookworm as builder
 
 WORKDIR /app
 COPY . .

crates/types/src/v1alpha1/wasmcloud_host_config.rs

@@ -1,8 +1,8 @@
-use k8s_openapi::api::core::v1::ResourceRequirements;
+use k8s_openapi::api::core::v1::{PodSpec, ResourceRequirements};
 use kube::CustomResource;
-use schemars::JsonSchema;
+use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{BTreeSet, HashMap};
 
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
 #[cfg_attr(test, derive(Default))]
@@ -36,10 +36,6 @@ pub struct WasmCloudHostConfigSpec {
     pub enable_structured_logging: Option<bool>,
     /// Name of a secret containing the registry credentials
     pub registry_credentials_secret: Option<String>,
-    /// Kubernetes resources to allocate for the host. See
-    /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for valid
-    /// values to use here.
-    pub resources: Option<WasmCloudHostConfigResources>,
     /// The control topic prefix to use for the host.
     pub control_topic_prefix: Option<String>,
     /// The leaf node domain to use for the NATS sidecar. Defaults to "leaf".
@@ -57,9 +53,39 @@ pub struct WasmCloudHostConfigSpec {
     /// The log level to use for the host. Defaults to "INFO".
     #[serde(default = "default_log_level")]
     pub log_level: String,
+    /// Kubernetes scheduling options for the wasmCloud host.
+    pub scheduling_options: Option<KubernetesSchedulingOptions>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
+pub struct KubernetesSchedulingOptions {
     /// Run hosts as a DaemonSet instead of a Deployment.
     #[serde(default)]
     pub daemonset: bool,
+    /// Kubernetes resources to allocate for the host. See
+    /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for valid
+    /// values to use here.
+    pub resources: Option<WasmCloudHostConfigResources>,
+    #[schemars(schema_with = "pod_schema")]
+    /// Any other pod template spec options to set for the underlying wasmCloud host pods.
+    pub pod_template_additions: Option<PodSpec>,
+}
+
+/// This is a workaround for the fact that we can't override the PodSpec schema to make containers
+/// an optional field. It generates the OpenAPI schema for the PodSpec type the same way that
+/// kube.rs does while dropping any required fields.
+fn pod_schema(_gen: &mut SchemaGenerator) -> Schema {
+    let gen = schemars::gen::SchemaSettings::openapi3()
+        .with(|s| {
+            s.inline_subschemas = true;
+            s.meta_schema = None;
+        })
+        .with_visitor(kube::core::schema::StructuralSchemaRewriter)
+        .into_generator();
+    let mut val = gen.into_root_schema_for::<PodSpec>();
+    // Drop `containers` as a required field, along with any others.
+    val.schema.object.as_mut().unwrap().required = BTreeSet::new();
+    val.schema.into()
 }
 
 fn default_host_replicas() -> u32 {

sample.yaml

@@ -18,5 +18,23 @@ spec:
   secretName: cluster-secrets
   logLevel: INFO
   natsAddress: nats://nats-cluster.default.svc.cluster.local
-  # Enable the following to run the wasmCloud hosts as a DaemonSet
-  #daemonset: true
+  # Additional options to control how the underlying wasmCloud hosts are scheduled in Kubernetes.
+  # This includes setting resource requirements for the nats and wasmCloud host
+  # containers along with any additional pot template settings.
+  #schedulingOptions:
+    # Enable the following to run the wasmCloud hosts as a DaemonSet
+    #daemonset: true
+    # Set the resource requirements for the nats and wasmCloud host containers.
+    #resources:
+    #  nats:
+    #    requests:
+    #      cpu: 100m
+    # wasmCloudHost:
+    #    requests:
+    #      cpu: 100m
+    # Any additional pod template settings to apply to the wasmCloud host pods.
+    # See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#podspec-v1-core for all valid options.
+    # Note that you *cannot* set the `containers` field here as it is managed by the controller.
+    #pod_template_additions:
+    #  nodeSelector:
+    #    kubernetes.io/os: linux

src/controller.rs

@@ -315,9 +315,11 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc<Context>) -> PodTemplate
 
     let mut nats_resources: Option<k8s_openapi::api::core::v1::ResourceRequirements> = None;
     let mut wasmcloud_resources: Option<k8s_openapi::api::core::v1::ResourceRequirements> = None;
-    if let Some(resources) = &config.spec.resources {
-        nats_resources = resources.nats.clone();
-        wasmcloud_resources = resources.wasmcloud.clone();
+    if let Some(scheduling_options) = &config.spec.scheduling_options {
+        if let Some(resources) = &scheduling_options.resources {
+            nats_resources = resources.nats.clone();
+            wasmcloud_resources = resources.wasmcloud.clone();
+        }
     }
 
     let containers = vec![
@@ -371,14 +373,34 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc<Context>) -> PodTemplate
             ..Default::default()
         },
     ];
-    PodTemplateSpec {
+
+    let mut volumes = vec![
+        Volume {
+            name: "nats-config".to_string(),
+            config_map: Some(ConfigMapVolumeSource {
+                name: Some(config.name_any()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        },
+        Volume {
+            name: "nats-creds".to_string(),
+            secret: Some(SecretVolumeSource {
+                secret_name: Some(config.spec.secret_name.clone()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        },
+    ];
+    let service_account = config.name_any();
+    let mut template = PodTemplateSpec {
         metadata: Some(ObjectMeta {
             labels: Some(labels),
             ..Default::default()
         }),
         spec: Some(PodSpec {
             service_account: Some(config.name_any()),
-            containers,
+            containers: containers.clone(),
             volumes: Some(vec![
                 Volume {
                     name: "nats-config".to_string(),
@@ -399,7 +421,21 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc<Context>) -> PodTemplate
             ]),
             ..Default::default()
         }),
-    }
+    };
+
+    if let Some(scheduling_options) = &config.spec.scheduling_options {
+        if let Some(pod_overrides) = &scheduling_options.pod_template_additions {
+            let mut overrides = pod_overrides.clone();
+            overrides.service_account_name = Some(service_account);
+            overrides.containers = containers.clone();
+            if let Some(vols) = overrides.volumes {
+                volumes.extend(vols);
+            }
+            overrides.volumes = Some(volumes);
+            template.spec = Some(overrides);
+        }
+    };
+    template
 }
 
 fn deployment_spec(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> DeploymentSpec {
@@ -504,31 +540,34 @@ async fn configure_hosts(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> Res
         ];
     }
 
-    if config.spec.daemonset {
-        let mut spec = daemonset_spec(config, ctx.clone());
-        spec.template.spec.as_mut().unwrap().containers[1]
-            .env
-            .as_mut()
-            .unwrap()
-            .append(&mut env_vars);
-        let ds = DaemonSet {
-            metadata: ObjectMeta {
-                name: Some(config.name_any()),
-                namespace: Some(config.namespace().unwrap()),
-                owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]),
+    if let Some(scheduling_options) = &config.spec.scheduling_options {
+        if scheduling_options.daemonset {
+            let mut spec = daemonset_spec(config, ctx.clone());
+            spec.template.spec.as_mut().unwrap().containers[1]
+                .env
+                .as_mut()
+                .unwrap()
+                .append(&mut env_vars);
+            let ds = DaemonSet {
+                metadata: ObjectMeta {
+                    name: Some(config.name_any()),
+                    namespace: Some(config.namespace().unwrap()),
+                    owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]),
+                    ..Default::default()
+                },
+                spec: Some(spec),
                 ..Default::default()
-            },
-            spec: Some(spec),
-            ..Default::default()
-        };
-
-        let api = Api::<DaemonSet>::namespaced(ctx.client.clone(), &config.namespace().unwrap());
-        api.patch(
-            &config.name_any(),
-            &PatchParams::apply(CLUSTER_CONFIG_FINALIZER),
-            &Patch::Apply(ds),
-        )
-        .await?;
+            };
+
+            let api =
+                Api::<DaemonSet>::namespaced(ctx.client.clone(), &config.namespace().unwrap());
+            api.patch(
+                &config.name_any(),
+                &PatchParams::apply(CLUSTER_CONFIG_FINALIZER),
+                &Patch::Apply(ds),
+            )
+            .await?;
+        }
     } else {
         let mut spec = deployment_spec(config, ctx.clone());
         spec.template.spec.as_mut().unwrap().containers[1]