From 97104b1842ef31cd457b22686cc4a5c56eee43e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Wei=C3=9Fe?= Date: Fri, 15 Nov 2024 11:03:18 +0100 Subject: [PATCH] Refactor marble-injector test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Weiße --- injector/injector_test.go | 499 ++++++++++++++++++-------------------- 1 file changed, 231 insertions(+), 268 deletions(-) diff --git a/injector/injector_test.go b/injector/injector_test.go index 0d14f885..34987238 100644 --- a/injector/injector_test.go +++ b/injector/injector_test.go @@ -11,21 +11,18 @@ import ( "strings" "testing" + "github.com/edgelesssys/marblerun/util/k8sutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" v1 "k8s.io/api/admission/v1" ) -func TestMutatesValidRequest(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - - rawJSON := `{ +func TestMutate(t *testing.T) { + defaultRequest := `{ "apiVersion": "admission.k8s.io/v1", "kind": "AdmissionReview", "request": { - "uid": "705ab4f5-6393-11e8-b7cc-42010a800002", "namespace": "injectable", "operation": "CREATE", "object": { @@ -33,7 +30,6 @@ func TestMutatesValidRequest(t *testing.T) { "apiVersion": "v1", "metadata": { "name": "testpod", - "creationTimestamp": null, "labels": { "name": "testpod", "marblerun/marbletype": "test", @@ -48,10 +44,7 @@ func TestMutatesValidRequest(t *testing.T) { "image": "test:image", "command": [ "/bin/bash" - ], - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "imagePullPolicy": "IfNotPresent" + ] }, { "name": "marble-test", @@ -62,278 +55,248 @@ func TestMutatesValidRequest(t *testing.T) { "imagePullPolicy": "IfNotPresent" } ] - }, - "status": {} - }, - "oldObject": null, - "dryRun": false, - "options": { - "kind": "CreateOptions", - "apiVersion": "meta.k8s.io/v1" + } } } }` - // test if patch contains all desired values - m := New("coordinator-mesh-api.marblerun:2001", "cluster.local", "kubernetes.azure.com/sgx_epc_mem_in_MiB", zaptest.NewLogger(t)) - response, err := m.mutate([]byte(rawJSON)) - require.NoError(err, "failed to mutate request") - - r := v1.AdmissionReview{} - require.NoError(json.Unmarshal(response, &r), "failed to unmarshal response with error %s", err) - - assert.Contains(string(r.Response.Patch), `"op":"add","path":"/spec/containers/1/resources","value":{"limits":{"kubernetes.azure.com/sgx_epc_mem_in_MiB":"10"}}`, "applied incorrect resource patch") - assert.Contains(string(r.Response.Patch), `"name":"EDG_MARBLE_COORDINATOR_ADDR","value":"coordinator-mesh-api.marblerun:2001"`, "failed to apply coordinator env variable patch") - assert.Contains(string(r.Response.Patch), `"name":"EDG_MARBLE_TYPE","value":"test"`, "failed to apply marble type env variable patch") - assert.Contains(string(r.Response.Patch), `"name":"EDG_MARBLE_DNS_NAMES","value":"test,test.injectable,test.injectable.svc.cluster.local"`, "failed to apply DNS name env variable patch") - assert.Contains(string(r.Response.Patch), `"name":"EDG_MARBLE_UUID_FILE"`, "failed to apply marble UUID file env variable patch") - assert.Contains(string(r.Response.Patch), `"op":"add","path":"/spec/containers/1/volumeMounts"`, "failed to apply volumeMount patch") - assert.Contains(string(r.Response.Patch), `"op":"add","path":"/spec/volumes"`, "failed to apply volumes patch") - assert.Contains(string(r.Response.Patch), `"op":"add","path":"/spec/tolerations","value":[{`, "failed to apply tolerations patch") - - assert.NotContains(string(r.Response.Patch), `"path":"/spec/containers/0/env`, "injected env variables into wrong pod") - assert.NotContains(string(r.Response.Patch), `"path":"/spec/containers/0/volumeMounts`, "injected volume mount into wrong pod") - assert.Contains(string(r.Response.Patch), `"path":"/spec/containers/0/resources","value":{}}`, "injected resources into the wrong pod") - - // test if patch works without sgx values - response, err = m.mutate([]byte(strings.Replace(rawJSON, `"marblerun/resource-injection": "enabled"`, `"marblerun/resource-injection": "disabled"`, -1))) - require.NoError(err, "failed to mutate request") - require.NoError(json.Unmarshal(response, &r), "failed to unmarshal response with error %s", err) - assert.NotContains(string(r.Response.Patch), `"op":"add","path":"/spec/containers/1/resources","value":{"limits":{"kubernetes.azure.com/sgx_epc_mem_in_MiB":"10"}}`, "patch contained sgx resources, but resources were not supposed to be set") -} - -func TestPreSetValues(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - - rawJSON := `{ - "apiVersion": "admission.k8s.io/v1", - "kind": "AdmissionReview", - "request": { - "uid": "705ab4f5-6393-11e8-b7cc-42010a800002", - "namespace": "injectable", - "operation": "CREATE", - "object": { - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "testpod", - "namespace": "injectable", - "creationTimestamp": null, - "labels": { - "name": "testpod", - "marblerun/marbletype": "test" - } - }, - "spec": { - "containers": [ - { - "name": "testpod", - "image": "test:image", - "command": [ - "/bin/bash" - ], - "resources": { - "requests": { - "cpu": 500 - }, - "limits": { - "cpu": 1000 + testCases := map[string]struct { + rawRequest string + resourceKey string + wantError bool + assertions func(assert *assert.Assertions, response *v1.AdmissionResponse) + }{ + "mutates valid request": { + rawRequest: defaultRequest, + resourceKey: k8sutil.AzureEpc.String(), + assertions: func(assert *assert.Assertions, response *v1.AdmissionResponse) { + assert.Contains(string(response.Patch), `"op":"add","path":"/spec/containers/1/resources","value":{"limits":{"kubernetes.azure.com/sgx_epc_mem_in_MiB":"10"}}`, "applied incorrect resource patch") + assert.Contains(string(response.Patch), `"name":"EDG_MARBLE_COORDINATOR_ADDR","value":"coordinator-mesh-api.marblerun:2001"`, "failed to apply coordinator env variable patch") + assert.Contains(string(response.Patch), `"name":"EDG_MARBLE_TYPE","value":"test"`, "failed to apply marble type env variable patch") + assert.Contains(string(response.Patch), `"name":"EDG_MARBLE_DNS_NAMES","value":"test,test.injectable,test.injectable.svc.cluster.local"`, "failed to apply DNS name env variable patch") + assert.Contains(string(response.Patch), `"name":"EDG_MARBLE_UUID_FILE"`, "failed to apply marble UUID file env variable patch") + assert.Contains(string(response.Patch), `"op":"add","path":"/spec/containers/1/volumeMounts"`, "failed to apply volumeMount patch") + assert.Contains(string(response.Patch), `"op":"add","path":"/spec/volumes"`, "failed to apply volumes patch") + assert.Contains(string(response.Patch), `"op":"add","path":"/spec/tolerations","value":[{`, "failed to apply tolerations patch") + assert.NotContains(string(response.Patch), `"path":"/spec/containers/0/env`, "injected env variables into wrong pod") + assert.NotContains(string(response.Patch), `"path":"/spec/containers/0/volumeMounts`, "injected volume mount into wrong pod") + assert.Contains(string(response.Patch), `"path":"/spec/containers/0/resources","value":{}}`, "injected resources into the wrong pod") + }, + }, + "resource injection can be disabled": { + rawRequest: strings.ReplaceAll(defaultRequest, `"marblerun/resource-injection": "enabled"`, `"marblerun/resource-injection": "disabled"`), + resourceKey: k8sutil.AzureEpc.String(), + assertions: func(assert *assert.Assertions, response *v1.AdmissionResponse) { + assert.NotContains( + string(response.Patch), + `"op":"add","path":"/spec/containers/1/resources","value":{"limits":{"kubernetes.azure.com/sgx_epc_mem_in_MiB":"10"}}`, + "patch contained sgx resources, but resources were not supposed to be set", + ) + }, + }, + "sgx resources are appended to existing resources - azure-plugin": { + rawRequest: `{ + "apiVersion": "admission.k8s.io/v1", + "kind": "AdmissionReview", + "request": { + "namespace": "injectable","operation": "CREATE", + "object": { + "kind":"Pod","apiVersion":"v1","metadata":{"name":"testpod","namespace":"injectable","labels":{"name":"testpod","marblerun/marbletype":"test"}}, + "spec": { + "containers": [ + { + "name": "testpod", + "image": "test:image", + "command": [ + "/bin/bash" + ], + "resources": { + "requests": { + "cpu": 500 + }, + "limits": { + "cpu": 1000 + } + } } - }, - "imagePullPolicy": "IfNotPresent", - "env": [ + ]}}}}`, + resourceKey: k8sutil.AzureEpc.String(), + assertions: func(assert *assert.Assertions, response *v1.AdmissionResponse) { + assert.Contains( + string(response.Patch), + `"op":"add","path":"/spec/containers/0/resources/limits/kubernetes.azure.com~1sgx_epc_mem_in_MiB","value":"10"}`, + "applied incorrect resource patch", + ) + }, + }, + "sgx resources are appended to existing resources - intel-plugin": { + rawRequest: `{ + "apiVersion": "admission.k8s.io/v1", + "kind": "AdmissionReview", + "request": { + "namespace": "injectable","operation": "CREATE", + "object": { + "kind":"Pod","apiVersion":"v1","metadata":{"name":"testpod","namespace":"injectable","labels":{"name":"testpod","marblerun/marbletype":"test"}}, + "spec": { + "containers": [ { - "name": "EDG_MARBLE_COORDINATOR_ADDR", - "value": "coordinator-mesh-api.marblerun:42" - }, + "name": "testpod", + "image": "test:image", + "command": [ + "/bin/bash" + ], + "resources": { + "requests": { + "cpu": 500 + }, + "limits": { + "cpu": 1000 + } + } + } + ]}}}}`, + resourceKey: k8sutil.IntelEpc.String(), + assertions: func(assert *assert.Assertions, response *v1.AdmissionResponse) { + assert.Contains(string(response.Patch), `"op":"add","path":"/spec/containers/0/resources/limits/sgx.intel.com~1epc","value":"10Mi"}`, "applied incorrect epc patch") + assert.Contains(string(response.Patch), `"op":"add","path":"/spec/containers/0/resources/limits/sgx.intel.com~1enclave","value":"1"}`, "applied incorrect enclave patch") + assert.Contains(string(response.Patch), `"op":"add","path":"/spec/containers/0/resources/limits/sgx.intel.com~1provision","value":"1"}`, "applied incorrect provision patch") + }, + }, + "env variables are ignored if already set": { + rawRequest: `{ + "apiVersion": "admission.k8s.io/v1", + "kind": "AdmissionReview", + "request": { + "namespace": "injectable","operation": "CREATE", + "object": { + "kind":"Pod","apiVersion":"v1","metadata":{"name":"testpod","namespace":"injectable","labels":{"name":"testpod","marblerun/marbletype":"test"}}, + "spec": { + "containers": [ { - "name": "EDG_MARBLE_TYPE", - "value": "different" - }, + "name": "testpod", + "image": "test:image", + "command": [ + "/bin/bash" + ], + "env": [ + { + "name": "EDG_MARBLE_COORDINATOR_ADDR", + "value": "coordinator-mesh-api.marblerun:42" + }, + { + "name": "EDG_MARBLE_TYPE", + "value": "different" + }, + { + "name": "EDG_MARBLE_DNS_NAMES", + "value": "different.example.com" + }, + { + "name": "EDG_MARBLE_UUID_FILE", + "value": "012345-678-90" + } + ] + } + ]}}}}`, + resourceKey: k8sutil.AzureEpc.String(), + assertions: func(assert *assert.Assertions, response *v1.AdmissionResponse) { + assert.NotContains(string(response.Patch), `"op":"add","path":"/spec/containers/0/env"`, "applied env variable patch when it shouldn't have") + }, + }, + "uuid mounts are added to existing mounts": { + rawRequest: `{ + "apiVersion": "admission.k8s.io/v1", + "kind": "AdmissionReview", + "request": { + "namespace":"injectable","operation":"CREATE", + "object": { + "kind":"Pod","apiVersion":"v1", + "metadata":{ + "name":"testpod", + "labels":{"name":"testpod","marblerun/marbletype":"test","marblerun/marblecontainer":"marble-test","marblerun/resource-injection":"enabled"} + }, + "spec": { + "containers": [ { - "name": "EDG_MARBLE_DNS_NAMES", - "value": "different.example.com" - }, + "name":"marble-test","image":"test:image","command":["/bin/bash"], + "volumeMounts": [ + { + "mountPath": "/test-uid", + "name": "test-uid" + } + ] + } + ], + "volumes": [ { - "name": "EDG_MARBLE_UUID_FILE", - "value": "012345-678-90" + "name": "marble-test-uid", + "volumeSource": { + "hostPath": { + "path": "/test-uid" + } + } } ] - } - ] - }, - "status": {} + }}}}`, + resourceKey: k8sutil.AzureEpc.String(), + assertions: func(assert *assert.Assertions, response *v1.AdmissionResponse) { + assert.NotContains(string(response.Patch), `"path":"/spec/containers/0/volumeMounts`) + assert.NotContains(string(response.Patch), `"op":"add","path":"/spec/volumes"`) }, - "oldObject": null, - "dryRun": false, - "options": { - "kind": "CreateOptions", - "apiVersion": "meta.k8s.io/v1" - } - } - }` - - m := New("coordinator-mesh-api.marblerun:2001", "cluster.local", "kubernetes.azure.com/sgx_epc_mem_in_MiB", zaptest.NewLogger(t)) - response, err := m.mutate([]byte(rawJSON)) - require.NoError(err, "failed to mutate request") - - r := v1.AdmissionReview{} - require.NoError(json.Unmarshal(response, &r), "failed to unmarshal response with error %s", err) - - assert.Contains(string(r.Response.Patch), `"op":"add","path":"/spec/containers/0/resources/limits/kubernetes.azure.com~1sgx_epc_mem_in_MiB","value":"10"}`, "applied incorrect resource patch") - assert.NotContains(string(r.Response.Patch), `"op":"add","path":"/spec/containers/0/env"`, "applied env variable patch when it shouldnt have") -} - -func TestRejectsUnsetMarbletype(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - - rawJSON := `{ - "apiVersion": "admission.k8s.io/v1", - "kind": "AdmissionReview", - "request": { - "uid": "705ab4f5-6393-11e8-b7cc-42010a800002", - "namespace": "injectable", - "operation": "CREATE", - "object": { - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "testpod", - "namespace": "injectable", - "creationTimestamp": null, - "labels": { - "name": "testpod" - } - }, - "spec": { - "containers": [ - { - "name": "testpod", - "image": "test:image", - "command": [ - "/bin/bash" - ], - "imagePullPolicy": "IfNotPresent" - } - ] - }, - "status": {} + }, + "requests with unset marbletype are rejected": { + rawRequest: `{ + "apiVersion": "admission.k8s.io/v1", + "kind": "AdmissionReview", + "request": { + "namespace": "injectable","operation": "CREATE", + "object": { + "kind": "Pod","apiVersion": "v1", + "metadata": { + "name": "testpod","namespace": "injectable", + "labels": {"name": "testpod"} + }, + "spec": {"containers": [{"name": "testpod","image": "test:image","command": ["/bin/bash"]}]} + }}}`, + resourceKey: k8sutil.AzureEpc.String(), + assertions: func(assert *assert.Assertions, response *v1.AdmissionResponse) { + assert.False(response.Allowed) }, - "oldObject": null, - "dryRun": false, - "options": { - "kind": "CreateOptions", - "apiVersion": "meta.k8s.io/v1" - } - } - }` + }, + "errors on invalid request": { + rawRequest: `This should return Error`, + resourceKey: k8sutil.AzureEpc.String(), + wantError: true, + }, + "errors on invalid Pod spec": { + rawRequest: `{ + "apiVersion": "admission.k8s.io/v1", + "kind": "AdmissionReview", + "request": { + "object": "invalid" + } + }`, + resourceKey: k8sutil.AzureEpc.String(), + wantError: true, + }, + } - m := New("coordinator-mesh-api.marblerun:2001", "cluster.local", "kubernetes.azure.com/sgx_epc_mem_in_MiB", zaptest.NewLogger(t)) - response, err := m.mutate([]byte(rawJSON)) - require.NoError(err, "failed to mutate request") - - r := v1.AdmissionReview{} - require.NoError(json.Unmarshal(response, &r), "failed to unmarshal response with error %s", err) - assert.False(r.Response.Allowed) -} + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) -func TestErrorsOnInvalid(t *testing.T) { - require := require.New(t) - - rawJSON := `This should return Error` - - m := New("coordinator-mesh-api.marblerun:2001", "cluster.local", "kubernetes.azure.com/sgx_epc_mem_in_MiB", zaptest.NewLogger(t)) - _, err := m.mutate([]byte(rawJSON)) - require.Error(err, "did not fail on invalid request") -} - -func TestErrorsOnInvalidPod(t *testing.T) { - require := require.New(t) - - rawJSON := `{ - "apiVersion": "admission.k8s.io/v1", - "kind": "AdmissionReview", - "request": { - "object": "invalid" - } - }` - - m := New("coordinator-mesh-api.marblerun:2001", "cluster.local", "kubernetes.azure.com/sgx_epc_mem_in_MiB", zaptest.NewLogger(t)) - _, err := m.mutate([]byte(rawJSON)) - require.Error(err, "did not fail when sending invalid request") -} - -func TestDoesNotCreateDoubleVolumeMounts(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - - rawJSON := `{ - "apiVersion": "admission.k8s.io/v1", - "kind": "AdmissionReview", - "request": { - "uid": "705ab4f5-6393-11e8-b7cc-42010a800002", - "namespace": "injectable", - "operation": "CREATE", - "object": { - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "testpod", - "creationTimestamp": null, - "labels": { - "name": "testpod", - "marblerun/marbletype": "test", - "marblerun/marblecontainer": "marble-test", - "marblerun/resource-injection": "enabled" - } - }, - "spec": { - "containers": [ - { - "name": "marble-test", - "image": "test:image", - "command": [ - "/bin/bash" - ], - "imagePullPolicy": "IfNotPresent", - "volumeMounts": [ - { - "mountPath": "/test-uid", - "name": "test-uid" - } - ] - } - ], - "volumes": [ - { - "name": "marble-test-uid", - "volumeSource": { - "hostPath": { - "path": "/test-uid" - } - } - } - ] - }, - "status": {} - }, - "oldObject": null, - "dryRun": false, - "options": { - "kind": "CreateOptions", - "apiVersion": "meta.k8s.io/v1" + m := New("coordinator-mesh-api.marblerun:2001", "cluster.local", tc.resourceKey, zaptest.NewLogger(t)) + response, err := m.mutate([]byte(tc.rawRequest)) + if tc.wantError { + require.Error(err) + return } - } - }` - - m := New("coordinator-mesh-api.marblerun:2001", "cluster.local", "kubernetes.azure.com/sgx_epc_mem_in_MiB", zaptest.NewLogger(t)) - response, err := m.mutate([]byte(rawJSON)) - require.NoError(err) - r := v1.AdmissionReview{} - require.NoError(json.Unmarshal(response, &r)) - assert.NotContains(string(r.Response.Patch), `"path":"/spec/containers/0/volumeMounts`) - assert.NotContains(string(r.Response.Patch), `"op":"add","path":"/spec/volumes"`) + assert.NoError(err) + r := v1.AdmissionReview{} + require.NoError(json.Unmarshal(response, &r)) + tc.assertions(assert, r.Response) + }) + } }