From 2c233f8abf7c5d34358994c1c746e01078495b03 Mon Sep 17 00:00:00 2001 From: hime Date: Wed, 4 Sep 2024 01:01:12 +0000 Subject: [PATCH] Update branch manually. --- Makefile | 6 +- cmd/csi_driver/Dockerfile | 2 +- cmd/csi_driver/main.go | 2 +- cmd/sidecar_mounter/Dockerfile | 2 +- cmd/sidecar_mounter/gcsfuse_binary | 2 +- cmd/sidecar_mounter/main.go | 9 +- cmd/webhook/Dockerfile | 2 +- .../manage-validating_admission_policy.sh | 75 +++++ .../webhook/validating_admission_policy.yaml | 60 ++++ deploy/overlays/dev/node_pprof.yaml | 2 +- docs/metrics/metrics.md | 1 + docs/releases.md | 62 +++- docs/troubleshooting.md | 14 +- go.mod | 6 +- hack/verify-golint.sh | 4 +- pkg/csi_driver/node.go | 6 +- pkg/csi_driver/utils.go | 5 +- pkg/metrics/metrics.go | 94 +++--- pkg/sidecar_mounter/logger.go | 2 +- pkg/sidecar_mounter/sidecar_mounter.go | 43 ++- pkg/sidecar_mounter/sidecar_mounter_config.go | 1 + .../validating_admission_policy_test.go | 268 +++++++++++++++ test/e2e/e2e_test.go | 1 + test/e2e/main.go | 2 + test/e2e/run-e2e-ci.sh | 2 +- test/e2e/specs/istio-service-entry.yaml | 30 ++ test/e2e/specs/istio-sidecar.yaml | 22 ++ test/e2e/specs/specs.go | 83 ++--- test/e2e/testsuites/failed_mount.go | 3 +- test/e2e/testsuites/gcsfuse_integration.go | 50 ++- .../gcsfuse_integration_file_cache.go | 26 +- ...tegration_file_cache_parallel_downloads.go | 306 ++++++++++++++++++ test/e2e/testsuites/istio.go | 96 ++++-- test/e2e/testsuites/metrics.go | 58 +++- test/e2e/utils/handler.go | 23 +- test/e2e/utils/install-istio.sh | 41 +++ 36 files changed, 1190 insertions(+), 221 deletions(-) create mode 100755 deploy/base/webhook/manage-validating_admission_policy.sh create mode 100644 deploy/base/webhook/validating_admission_policy.yaml create mode 100644 pkg/webhook/validating_admission_policy_test.go create mode 100644 test/e2e/specs/istio-service-entry.yaml create mode 100644 test/e2e/specs/istio-sidecar.yaml create mode 100644 test/e2e/testsuites/gcsfuse_integration_file_cache_parallel_downloads.go create mode 100755 test/e2e/utils/install-istio.sh diff --git a/Makefile b/Makefile index 9dad9e995..4e55b13ce 100755 --- a/Makefile +++ b/Makefile @@ -21,7 +21,8 @@ export BUILD_ARM ?= false BINDIR ?= $(shell pwd)/bin GCSFUSE_PATH ?= $(shell cat cmd/sidecar_mounter/gcsfuse_binary) LDFLAGS ?= -s -w -X main.version=${STAGINGVERSION} -extldflags '-static' -PROJECT ?= $(shell gcloud config get-value project 2>&1 | head -n 1) +# assume that a GKE cluster identifier follows the format gke_{project-name}_{location}_{cluster-name} +PROJECT ?= $(shell kubectl config current-context | cut -d '_' -f 2) CA_BUNDLE ?= $(shell kubectl config view --raw -o json | jq '.clusters[]' | jq "select(.name == \"$(shell kubectl config current-context)\")" | jq '.cluster."certificate-authority-data"' | head -n 1) IDENTITY_PROVIDER ?= $(shell kubectl get --raw /.well-known/openid-configuration | jq -r .issuer) @@ -38,6 +39,7 @@ ifneq ("$(shell docker buildx build --help | grep 'provenance')", "") DOCKER_BUILDX_ARGS += --provenance=false endif +$(info PROJECT is ${PROJECT}) $(info OVERLAY is ${OVERLAY}) $(info STAGINGVERSION is ${STAGINGVERSION}) $(info DRIVER_IMAGE is ${DRIVER_IMAGE}) @@ -183,9 +185,11 @@ install: make generate-spec-yaml OVERLAY=${OVERLAY} REGISTRY=${REGISTRY} STAGINGVERSION=${STAGINGVERSION} kubectl apply -f ${BINDIR}/gcs-fuse-csi-driver-specs-generated.yaml ./deploy/base/webhook/create-cert.sh --namespace gcs-fuse-csi-driver --service gcs-fuse-csi-driver-webhook --secret gcs-fuse-csi-driver-webhook-secret + ./deploy/base/webhook/manage-validating_admission_policy.sh --install uninstall: kubectl delete -k deploy/overlays/${OVERLAY} --wait + ./deploy/base/webhook/manage-validating_admission_policy.sh --uninstall generate-spec-yaml: mkdir -p ${BINDIR} diff --git a/cmd/csi_driver/Dockerfile b/cmd/csi_driver/Dockerfile index b720e45fa..576c14b05 100644 --- a/cmd/csi_driver/Dockerfile +++ b/cmd/csi_driver/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # Build driver go binary -FROM golang:1.22.6 AS driver-builder +FROM golang:1.23.0 AS driver-builder ARG STAGINGVERSION diff --git a/cmd/csi_driver/main.go b/cmd/csi_driver/main.go index f0e8c4cb4..9bc778b2c 100644 --- a/cmd/csi_driver/main.go +++ b/cmd/csi_driver/main.go @@ -45,7 +45,7 @@ var ( identityProvider = flag.String("identity-provider", "", "The Identity Provider to authenticate with GCS API.") enableProfiling = flag.Bool("enable-profiling", false, "enable the golang pprof at port 6060") informerResyncDurationSec = flag.Int("informer-resync-duration-sec", 1800, "informer resync duration in seconds") - metricsEndpoint = flag.String("metrics-endpoint", "", "The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means metrics endpoint is disabled.") + metricsEndpoint = flag.String("metrics-endpoint", "", "The TCP network address where the Prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means that the metrics endpoint is disabled.") // These are set at compile time. version = "unknown" diff --git a/cmd/sidecar_mounter/Dockerfile b/cmd/sidecar_mounter/Dockerfile index 71f78ffeb..b77927d45 100644 --- a/cmd/sidecar_mounter/Dockerfile +++ b/cmd/sidecar_mounter/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # Build sidecar-mounter go binary -FROM golang:1.22.6 AS sidecar-mounter-builder +FROM golang:1.23.0 AS sidecar-mounter-builder ARG STAGINGVERSION diff --git a/cmd/sidecar_mounter/gcsfuse_binary b/cmd/sidecar_mounter/gcsfuse_binary index bf4564565..ab4a61515 100644 --- a/cmd/sidecar_mounter/gcsfuse_binary +++ b/cmd/sidecar_mounter/gcsfuse_binary @@ -1 +1 @@ -gs://gke-release-staging/gcsfuse/v2.4.0-gke.0/gcsfuse_bin +gs://gke-release-staging/gcsfuse/v2.4.1-gke.0/gcsfuse_bin diff --git a/cmd/sidecar_mounter/main.go b/cmd/sidecar_mounter/main.go index 86a58d384..76a93ab4d 100644 --- a/cmd/sidecar_mounter/main.go +++ b/cmd/sidecar_mounter/main.go @@ -34,9 +34,10 @@ import ( ) var ( - gcsfusePath = flag.String("gcsfuse-path", "/gcsfuse", "gcsfuse path") - volumeBasePath = flag.String("volume-base-path", webhook.SidecarContainerTmpVolumeMountPath+"/.volumes", "volume base path") - _ = flag.Int("grace-period", 0, "grace period for gcsfuse termination. This flag has been deprecated, has no effect and will be removed in the future.") + gcsfusePath = flag.String("gcsfuse-path", "/gcsfuse", "gcsfuse path") + volumeBasePath = flag.String("volume-base-path", webhook.SidecarContainerTmpVolumeMountPath+"/.volumes", "volume base path") + metricsScrapeInterval = flag.Int("metrics-scrape-interval", 10, "Scrape interval in seconds for gcsfuse metrics endpoint.") + _ = flag.Int("grace-period", 0, "grace period for gcsfuse termination. This flag has been deprecated, has no effect and will be removed in the future.") // This is set at compile time. version = "unknown" ) @@ -62,7 +63,7 @@ func main() { time.Sleep(1500 * time.Millisecond) mc := sidecarmounter.NewMountConfig(sp) if mc != nil { - if err := mounter.Mount(ctx, mc); err != nil { + if err := mounter.Mount(ctx, mc, *metricsScrapeInterval); err != nil { mc.ErrWriter.WriteMsg(fmt.Sprintf("failed to mount bucket %q for volume %q: %v\n", mc.BucketName, mc.VolumeName, err)) } } diff --git a/cmd/webhook/Dockerfile b/cmd/webhook/Dockerfile index 4744ef601..0940e81de 100644 --- a/cmd/webhook/Dockerfile +++ b/cmd/webhook/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # Build webhook go binary -FROM golang:1.22.6 AS webhook-builder +FROM golang:1.23.0 AS webhook-builder ARG STAGINGVERSION diff --git a/deploy/base/webhook/manage-validating_admission_policy.sh b/deploy/base/webhook/manage-validating_admission_policy.sh new file mode 100755 index 000000000..ea163db56 --- /dev/null +++ b/deploy/base/webhook/manage-validating_admission_policy.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage() { + cat <= 1 && minorVersion >= 30 )) || (( majorVersion > 1 )); then + echo "Cluster version is greater than or equal to 1.30" + script_path=$(dirname "$(realpath "$0")") + + if ( $install == "true" ); then + echo "Installing ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding" + kubectl apply -f "${script_path}/validating_admission_policy.yaml" + fi + + if ( $uninstall == "true" ); then + echo "Uninstalling ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding" + kubectl delete -f "${script_path}/validating_admission_policy.yaml" + fi + + else + echo "Cluster version is less than 1.30, skip ValidatingAdmissionPolicy management" + fi +else + echo "Invalid version format: ${versionStr}" +fi diff --git a/deploy/base/webhook/validating_admission_policy.yaml b/deploy/base/webhook/validating_admission_policy.yaml new file mode 100644 index 000000000..52b9ecca3 --- /dev/null +++ b/deploy/base/webhook/validating_admission_policy.yaml @@ -0,0 +1,60 @@ +# Copyright 2018 The Kubernetes Authors. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: "gcsfuse-sidecar-validator.csi.storage.gke.io" +spec: + failurePolicy: Ignore # will not block other Pod requests + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + operations: ["CREATE"] + matchConditions: + - name: "include-pods-with-gcsfuse-volumes" + expression: 'has(object.metadata.annotations) && "gke-gcsfuse/volumes" in object.metadata.annotations && object.metadata.annotations["gke-gcsfuse/volumes"] == "true"' + - name: "include-pods-with-native-sidecar" + expression: 'has(object.spec.initContainers) && object.spec.initContainers.exists(c, c.name == "gke-gcsfuse-sidecar")' + variables: + - name: "sidecar" + expression: 'object.spec.initContainers.filter(c, c.name == "gke-gcsfuse-sidecar")[0]' + validations: + - messageExpression: '"the native gcsfuse sidecar init container must have restartPolicy:Always."' + reason: Invalid + expression: |- + has(variables.sidecar.restartPolicy) && + variables.sidecar.restartPolicy == "Always" + - messageExpression: '"the native gcsfuse sidecar init container must have env var NATIVE_SIDECAR with value TRUE."' + reason: Invalid + expression: |- + has(variables.sidecar.env) && + variables.sidecar.env.exists(e, e.name == "NATIVE_SIDECAR" && e.value == "TRUE") +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: "gcsfuse-sidecar-validator-binding.csi.storage.gke.io" +spec: + policyName: "gcsfuse-sidecar-validator.csi.storage.gke.io" + validationActions: [Deny] + matchResources: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + operations: ["CREATE"] diff --git a/deploy/overlays/dev/node_pprof.yaml b/deploy/overlays/dev/node_pprof.yaml index 6862500ca..2af8ba338 100755 --- a/deploy/overlays/dev/node_pprof.yaml +++ b/deploy/overlays/dev/node_pprof.yaml @@ -31,4 +31,4 @@ spec: - --metrics-endpoint=:9920 - --enable-profiling=true ports: - - containerPort: 6060 \ No newline at end of file + - containerPort: 6060 diff --git a/docs/metrics/metrics.md b/docs/metrics/metrics.md index ab79c8840..3aea0137d 100644 --- a/docs/metrics/metrics.md +++ b/docs/metrics/metrics.md @@ -116,6 +116,7 @@ In the CSI driver, each metric record includes the following extra labels so tha - namespace_name - volume_name - bucket_name +- pod_uid The Prometheus UI provides an easy interface to query and visualize metrics. See [Querying Prometheus documentation](https://prometheus.io/docs/prometheus/latest/querying/basics/) for details. diff --git a/docs/releases.md b/docs/releases.md index 6800b5a79..c17fcf408 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -19,22 +19,23 @@ limitations under the License. ## GKE Compatibility -| CSI Driver Version | Status | Release Date | Cloud Storage FUSE Version | Sidecar Container Image | Earliest GKE 1.27 | Earliest GKE 1.28 | Earliest GKE 1.29 | Earliest GKE 1.30 | -|--------------------------------------------------------------------------------------------|-----------|--------------|------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|---------------------|--------------------|--------------------| -| [v0.1.4](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.4) | Released | 2023-08-15 | [v1.0.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.0.0) | N/A | 1.27.2-gke.1200 | 1.28.1-gke.200 | None | None | -| [v0.1.6](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.6) | Released | 2023-10-25 | [v1.2.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.2.0) | N/A | 1.27.6-gke.1487000 | 1.28.2-gke.1078000 | None | None | -| [v0.1.7](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.7) | Released | 2023-11-25 | [v1.2.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.2.1) | N/A | 1.27.7-gke.1121000 | 1.28.3-gke.1260000 | None | None | -| [v0.1.8](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.8) | Released | 2023-12-08 | [v1.2.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.2.1) | N/A | 1.27.7-gke.1279000 | 1.28.3-gke.1430000 | 1.29.0-gke.1003000 | None | -| [v0.1.12](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.12) | Released | 2024-01-25 | [v1.4.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.4.0) | [7898e40bf57f](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:7898e40bf57f159dc828511f4217cb42c08fa4df0c9ad732a0b0747b66e415c6) | 1.27.9-gke.1092000 | None | 1.29.0-gke.1381000 | None | -| [v0.1.13](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.13) | Released | 2024-02-08 | [v1.4.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.4.1) | [972699a4bf89](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:972699a4bf8973f7614f09908412a1fca24ea939eac2d3fcca599109f71fc162) | 1.27.10-gke.1055000 | 1.28.6-gke.1095000 | 1.29.1-gke.1425000 | None | -| [v0.1.14](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.14) | Released | 2024-02-20 | [v1.4.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.4.1) | [c83609ecf50d](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:c83609ecf50d05a141167b8c6cf4dfe14ff07f01cd96a9790921db6748d40902) | 1.27.11-gke.1018000 | 1.28.6-gke.1456000 | 1.29.2-gke.1060000 | None | -| [v1.2.0](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.2.0) | Released | 2024-04-04 | [v2.0.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.0.0) | [31880114306b](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:31880114306b1fb5d9e365ae7d4771815ea04eb56f0464a514a810df9470f88f) | 1.27.12-gke.1190000 | 1.28.8-gke.1175000 | 1.29.3-gke.1093000 | 1.30.0-gke.1167000 | -| [v1.3.0](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.3.0) | Released | 2024-05-05 | [v2.0.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.0.1) | [1f36463e0827](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:1f36463e0827619ba8af3f94ee21d404b635573f70ea3d6447b26f29fd97b9f0) | 1.27.13-gke.1166000 | 1.28.9-gke.1209000 | 1.29.4-gke.1447000 | None | -| [v1.3.1](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.3.1) | Released | 2024-05-17 | [v2.0.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.0.1) | [17c8a585a1e0](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:17c8a585a1e088cb90827386958407d817d65595cd80be74bb48c0498eea7abd) | 1.27.14-gke.1011000 | 1.28.10-gke.1012000 | 1.29.5-gke.1010000 | 1.30.1-gke.1156000 | -| [v1.3.2](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.3.2) | Released | 2024-06-07 | [v2.1.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.1.0) | [46a32daec5df](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:46a32daec5df0688da1381876747c5f15b6b5d46dec01ea6cbfae1caf0a4366a) | None | None | 1.29.5-gke.1121000 | None | -| [v1.4.1](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.4.1) | Released | 2024-06-07 | [v2.2.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.2.0) | [26aaa3ec5955](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:26aaa3ec5955506ce36c4d9c0bae909401168b9a0f52d36661f66ca0789bea0e) | None | None | 1.29.5-gke.1198000 | 1.30.1-gke.1329000 | -| [v1.4.2](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.4.2) | Released | 2024-06-28 | [v2.3.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.3.1) | [80c2a52aaa16](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:80c2a52aaa16ee7d9956a4e4afb7442893919300af84ae445ced32ac758c55ad) | None | None | 1.29.6-gke.1157000 | 1.30.2-gke.1266000 | -| [v1.4.3](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.4.3) | Released | 2024-07-11 | [v2.3.2](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.3.2) | [7c74e9ef7c49](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:7c74e9ef7c49627c252087458fd65fa59811161134e5e9a6d3c16aaff0616174) | None | None | 1.29.6-gke.1342000 | 1.30.3-gke.1225000 | +| CSI Driver Version | Status | Release Date | Cloud Storage FUSE Version | Sidecar Container Image | Earliest GKE 1.27 | Earliest GKE 1.28 | Earliest GKE 1.29 | Earliest GKE 1.30 | +|--------------------------------------------------------------------------------------------|----------|--------------|------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|---------------------|--------------------|--------------------| +| [v0.1.4](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.4) | Released | 2023-08-15 | [v1.0.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.0.0) | N/A | 1.27.2-gke.1200 | 1.28.1-gke.200 | None | None | +| [v0.1.6](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.6) | Released | 2023-10-25 | [v1.2.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.2.0) | N/A | 1.27.6-gke.1487000 | 1.28.2-gke.1078000 | None | None | +| [v0.1.7](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.7) | Released | 2023-11-25 | [v1.2.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.2.1) | N/A | 1.27.7-gke.1121000 | 1.28.3-gke.1260000 | None | None | +| [v0.1.8](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.8) | Released | 2023-12-08 | [v1.2.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.2.1) | N/A | 1.27.7-gke.1279000 | 1.28.3-gke.1430000 | 1.29.0-gke.1003000 | None | +| [v0.1.12](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.12) | Released | 2024-01-25 | [v1.4.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.4.0) | [7898e40bf57f](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:7898e40bf57f159dc828511f4217cb42c08fa4df0c9ad732a0b0747b66e415c6) | 1.27.9-gke.1092000 | None | 1.29.0-gke.1381000 | None | +| [v0.1.13](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.13) | Released | 2024-02-08 | [v1.4.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.4.1) | [972699a4bf89](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:972699a4bf8973f7614f09908412a1fca24ea939eac2d3fcca599109f71fc162) | 1.27.10-gke.1055000 | 1.28.6-gke.1095000 | 1.29.1-gke.1425000 | None | +| [v0.1.14](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v0.1.14) | Released | 2024-02-20 | [v1.4.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v1.4.1) | [c83609ecf50d](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:c83609ecf50d05a141167b8c6cf4dfe14ff07f01cd96a9790921db6748d40902) | 1.27.11-gke.1018000 | 1.28.6-gke.1456000 | 1.29.2-gke.1060000 | None | +| [v1.2.0](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.2.0) | Released | 2024-04-04 | [v2.0.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.0.0) | [31880114306b](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:31880114306b1fb5d9e365ae7d4771815ea04eb56f0464a514a810df9470f88f) | 1.27.12-gke.1190000 | 1.28.8-gke.1175000 | 1.29.3-gke.1093000 | 1.30.0-gke.1167000 | +| [v1.3.0](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.3.0) | Released | 2024-05-05 | [v2.0.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.0.1) | [1f36463e0827](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:1f36463e0827619ba8af3f94ee21d404b635573f70ea3d6447b26f29fd97b9f0) | 1.27.13-gke.1166000 | 1.28.9-gke.1209000 | 1.29.4-gke.1447000 | None | +| [v1.3.1](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.3.1) | Released | 2024-05-17 | [v2.0.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.0.1) | [17c8a585a1e0](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:17c8a585a1e088cb90827386958407d817d65595cd80be74bb48c0498eea7abd) | 1.27.14-gke.1011000 | 1.28.10-gke.1012000 | 1.29.5-gke.1010000 | 1.30.1-gke.1156000 | +| [v1.3.2](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.3.2) | Released | 2024-06-07 | [v2.1.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.1.0) | [46a32daec5df](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:46a32daec5df0688da1381876747c5f15b6b5d46dec01ea6cbfae1caf0a4366a) | None | None | 1.29.5-gke.1121000 | None | +| [v1.4.1](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.4.1) | Released | 2024-06-07 | [v2.2.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.2.0) | [26aaa3ec5955](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:26aaa3ec5955506ce36c4d9c0bae909401168b9a0f52d36661f66ca0789bea0e) | None | None | 1.29.5-gke.1198000 | 1.30.1-gke.1329000 | +| [v1.4.2](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.4.2) | Released | 2024-06-28 | [v2.3.1](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.3.1) | [80c2a52aaa16](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:80c2a52aaa16ee7d9956a4e4afb7442893919300af84ae445ced32ac758c55ad) | None | None | 1.29.6-gke.1157000 | 1.30.2-gke.1266000 | +| [v1.4.3](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.4.3) | Released | 2024-07-11 | [v2.3.2](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.3.2) | [7c74e9ef7c49](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:7c74e9ef7c49627c252087458fd65fa59811161134e5e9a6d3c16aaff0616174) | None | None | 1.29.6-gke.1342000 | 1.30.3-gke.1225000 | +| [v1.5.0](https://github.com/GoogleCloudPlatform/gcs-fuse-csi-driver/releases/tag/v1.5.0) | Released | 2024-08-19 | [v2.4.0](https://github.com/GoogleCloudPlatform/gcsfuse/releases/tag/v2.4.0) | [a527a083127f](https://gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:a527a083127fb456c96a6e4a478639222065dc0c2d485729e63605035d624f8f) | None | None | None | 1.30.3-gke.1639000 | > Note: The above GKE versions may not be valid any more, please follow the [GKE documentation](https://cloud.google.com/kubernetes-engine/docs/concepts/release-channels#what_versions_are_available_in_a_channel) to check what versions are available in a channel. The release versions that are not included in the above table are deprecated or abandoned. @@ -42,6 +43,35 @@ The new CSI driver version will be first available in GKE Rapid channel on its r ## Releases +### v1.5.0 + +* Update gcsfuse to v2.1.0 by @msau42 in +* Use rapid channel for 1.30 by @hime in +* introduce a new PV volume attribute to skip bucket access check in CSI by @saikat-royc in +* update failed_mount e2e testcases for skip bucket access knob by @saikat-royc in +* Update gcsfuse to v2.2.0 by @msau42 in +* Remove skip access check from mount options by @saikat-royc in +* Add success mounts e2e tests for skip bucket access skip flag by @saikat-royc in +* fix metadataCacheTTLSeconds typo by @saikat-royc in +* support metadataCacheTtlSeconds volume attribute by @songjiaxun in +* Bump golang from 1.22.3 to 1.22.4 in /cmd/sidecar_mounter by @dependabot in +* Bump golang from 1.22.3 to 1.22.4 in /cmd/csi_driver by @dependabot in +* Bump golang from 1.22.3 to 1.22.4 in /cmd/webhook by @dependabot in +* Bump github.com/huandu/xstrings from 1.4.0 to 1.5.0 by @dependabot in +* Update integration tests to go1.22.4 by @hime in +* Updated playbook for IO error with workload identity federation by @saikat-royc in +* add example for sub-directory mounts by @saikat-royc in +* update gcsfuse binary to 2.3.0 by @saikat-royc in +* update gcsfuse to 2.3.1 by @saikat-royc in +* use Pod informer to avoid Pod GET API calls by @songjiaxun in +* Bump golang from 1.22.4 to 1.22.5 in /cmd/sidecar_mounter by @dependabot in +* Bump golang from 1.22.4 to 1.22.5 in /cmd/csi_driver by @dependabot in +* Bump golang from 1.22.4 to 1.22.5 in /cmd/webhook by @dependabot in +* Remove gcsfuse arm binary dependency for building csi driver. by @hime in +* update gcsfuse binary to 2.3.2 by @saikat-royc in +* Update release and compatibility table. by @hime in +* cache NodePublishVolume response state for republish by @songjiaxun in + ### v1.4.3 * update gcsfuse binary to 2.3.2 by @saikat-royc in diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index abc12d580..7d7691993 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -298,9 +298,9 @@ The Cloud Storage FUSE [stat metadata cache](https://cloud.google.com/storage/do - Mount options: > Note: The following mount options are being deprecated, and you cannot configure type cache capacity using mount options. We recommand upgrading your GKE clusters to a newer version, and using volume attributes to configure metadata caches. - - `stat-cache-capacity`: Set the value to `"-1"` to let the stat cache use as much memory as needed. - - `stat-cache-ttl`: Set the value to `"-1"` to bypass a TTL expiration and serve the file from the cache whenever it's available. - - `type-cache-ttl`: Set the value to `"-1"` to bypass a TTL expiration and serve the file from the cache whenever it's available. + - `stat-cache-capacity`: Specifies the number of entries that the stat cache can hold. This impacts memory consumption. The default value is 4096. + - `stat-cache-ttl`: Specifies how long to cache StatObject results and inode attributes. The default value is 60s. + - `type-cache-ttl`: Specifies how long to cache the mapping between names and files or directories in directory inodes. The default value is 60s. - For example: - Inline ephemeral volume @@ -315,7 +315,7 @@ The Cloud Storage FUSE [stat metadata cache](https://cloud.google.com/storage/do driver: gcsfuse.csi.storage.gke.io volumeAttributes: bucketName: - mountOptions: "stat-cache-capacity=-1,stat-cache-ttl=-1,type-cache-ttl=-1" + mountOptions: "stat-cache-capacity=102400,stat-cache-ttl=3600s,type-cache-ttl=3600s" ``` - PersistentVolume @@ -326,9 +326,9 @@ The Cloud Storage FUSE [stat metadata cache](https://cloud.google.com/storage/do spec: ... mountOptions: - - stat-cache-capacity=-1 - - stat-cache-ttl=-1 - - type-cache-ttl=-1 + - stat-cache-capacity=102400 + - stat-cache-ttl=3600s + - type-cache-ttl=3600s csi: driver: gcsfuse.csi.storage.gke.io volumeHandle: diff --git a/go.mod b/go.mod index 30f4cc75d..f0f79e6d6 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/googlecloudplatform/gcs-fuse-csi-driver -go 1.22.4 - -toolchain go1.22.6 +go 1.23.0 require ( cloud.google.com/go/compute/metadata v0.5.0 @@ -27,6 +25,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.30.3 k8s.io/apimachinery v0.30.3 + k8s.io/apiserver v0.30.3 k8s.io/client-go v0.30.3 k8s.io/klog/v2 v2.130.1 k8s.io/kubernetes v1.30.3 @@ -141,7 +140,6 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apiextensions-apiserver v0.30.3 // indirect - k8s.io/apiserver v0.30.3 // indirect k8s.io/cloud-provider v0.30.3 // indirect k8s.io/component-base v0.30.3 // indirect k8s.io/component-helpers v0.30.3 // indirect diff --git a/hack/verify-golint.sh b/hack/verify-golint.sh index bb18b0e6d..51aa3452b 100755 --- a/hack/verify-golint.sh +++ b/hack/verify-golint.sh @@ -18,7 +18,7 @@ set -o errexit set -o nounset set -o pipefail -TOOL_VERSION="v1.59.1" +TOOL_VERSION="v1.60.1" export PATH=$PATH:$(go env GOPATH)/bin go install "github.com/golangci/golangci-lint/cmd/golangci-lint@${TOOL_VERSION}" @@ -34,6 +34,6 @@ golangci-lint run --no-config --timeout=10m --sort-results \ --enable-all \ --max-same-issues 100 \ --disable exhaustruct,gomnd,lll,gochecknoglobals,funlen,varnamelen,wsl,testpackage,wrapcheck,err113,ireturn,gocyclo,cyclop,godox,gocognit,nestif,gomoddirectives,maintidx,depguard,mnd,execinquery \ ---go 1.22.6 # the builder version +--go 1.23.0 # the builder version echo "Congratulations! Lint check completed for all Go source files." diff --git a/pkg/csi_driver/node.go b/pkg/csi_driver/node.go index 895c564b2..f655de19f 100644 --- a/pkg/csi_driver/node.go +++ b/pkg/csi_driver/node.go @@ -163,7 +163,7 @@ func (s *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublish // notify the sidecar container to exit. if !isInitContainer { if err := putExitFile(pod, targetPath); err != nil { - return nil, status.Errorf(codes.Internal, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) } } @@ -176,13 +176,13 @@ func (s *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublish return &csi.NodePublishVolumeResponse{}, nil } - return nil, status.Errorf(code, err.Error()) + return nil, status.Error(code, err.Error()) } // Check if there is any error from the sidecar container code, err = checkSidecarContainerErr(isInitContainer, pod) if code != codes.OK { - return nil, status.Errorf(code, err.Error()) + return nil, status.Error(code, err.Error()) } // TODO: Check if the socket listener timed out diff --git a/pkg/csi_driver/utils.go b/pkg/csi_driver/utils.go index 84b2102cc..0d23e5bc5 100644 --- a/pkg/csi_driver/utils.go +++ b/pkg/csi_driver/utils.go @@ -306,7 +306,7 @@ func putExitFile(pod *corev1.Pod, targetPath string) error { // Check if all the containers besides the sidecar container exited if podRestartPolicyIsOnFailure || podRestartPolicyIsNever || podIsTerminating { - if pod.Status.ContainerStatuses == nil || len(pod.Status.ContainerStatuses) == 0 { + if len(pod.Status.ContainerStatuses) == 0 { return nil } @@ -386,7 +386,8 @@ func checkGcsFuseErr(isInitContainer bool, pod *corev1.Pod, targetPath string) ( if err == nil && len(errMsg) > 0 { errMsgStr := string(errMsg) code := codes.Internal - if strings.Contains(errMsgStr, "Incorrect Usage") { + if strings.Contains(errMsgStr, "Incorrect Usage") || + strings.Contains(errMsgStr, "unknown flag") { code = codes.InvalidArgument } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 2ecea851f..7b96dd604 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -1,6 +1,6 @@ /* Copyright 2018 The Kubernetes Authors. -Copyright 2022 Google LLC +Copyright 2024 Google LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,12 +18,13 @@ limitations under the License. package metrics import ( - "bufio" + "errors" "fmt" "net/http" "os" + "path/filepath" + "regexp" "strconv" - "strings" "time" "github.com/googlecloudplatform/gcs-fuse-csi-driver/pkg/util" @@ -36,8 +37,9 @@ import ( const ( metricsPath = "/metrics" - metricsFileNameTemplate = `/metrics_%d.prom` - generationFileName = "/generation.txt" + metricsFileRegex = `metrics_(\d+)\.prom` + metricsFileNameTemplate = `metrics_%d.prom` + metricsTempFileName = "temp.prom" ) type Manager interface { @@ -99,7 +101,7 @@ func (mm *manager) RegisterMetricsCollector(targetPath, podNamespace, podName, b "bucket_name": bucketName, "pod_uid": podUID, }) - if err := mm.registry.Register(c); err != nil && !strings.Contains(err.Error(), prometheus.AlreadyRegisteredError{}.Error()) { + if err := mm.registry.Register(c); err != nil && !errors.Is(err, prometheus.AlreadyRegisteredError{}) { klog.Errorf("failed to register metrics collector for pod %v/%v, volume %q, bucket %q: %v", podNamespace, podName, volumeName, bucketName, err) } } @@ -143,9 +145,16 @@ func (c *textFileCollector) Describe(ch chan<- *prometheus.Desc) { // Collect emits metrics. func (c *textFileCollector) Collect(ch chan<- prometheus.Metric) { - families, err := ProcessMetricsFile(c.path) + metricsFilePath, err := GetMetricsFilePath(c.path) if err != nil { - klog.Errorf("failed to process metrics from metrics file: %v", err) + klog.Warningf("failed to get metrics file: %s", err.Error()) + + return + } + + families, err := ProcessMetricsFile(metricsFilePath) + if err != nil { + klog.Errorf("failed to process metrics file: %v", err) return } @@ -157,16 +166,7 @@ func (c *textFileCollector) Collect(ch chan<- prometheus.Metric) { // ProcessMetricsFile processes a metrics file that follows Prometheus text format: https://prometheus.io/docs/instrumenting/exposition_formats/, // returning its MetricFamily. -func ProcessMetricsFile(directoryPath string) (map[string]*dto.MetricFamily, error) { - // Find latest file generation. - generationFilePath := directoryPath + generationFileName - genNumber, err := getGenerationNumber(generationFilePath) - if err != nil { - return nil, fmt.Errorf("could not get generation number from %s: %w", generationFilePath, err) - } - - // Find latest metrics file name and open. - metricsFilePath := directoryPath + GetMetricsFileName(genNumber) +func ProcessMetricsFile(metricsFilePath string) (map[string]*dto.MetricFamily, error) { metricsFile, err := os.Open(metricsFilePath) if err != nil { return nil, fmt.Errorf("failed to open metrics file %q: %w", metricsFilePath, err) @@ -182,38 +182,60 @@ func ProcessMetricsFile(directoryPath string) (map[string]*dto.MetricFamily, err return metricFamilies, nil } +func GetMetricsFilePath(directoryPath string) (string, error) { + // Find latest file generation. + genNumber, err := getGenerationNumber(directoryPath) + if err != nil { + return "", fmt.Errorf("could not get generation number from %s: %w", directoryPath, err) + } + + // Find latest metrics file name and open. + metricsFilePath := filepath.Join(directoryPath, GetMetricsFileName(genNumber)) + + return metricsFilePath, nil +} + // GetMetricsFilePath creates the expected name for the latest metrics file. func GetMetricsFileName(genNumber int) string { return fmt.Sprintf(metricsFileNameTemplate, genNumber) } -func GetGenerationFileName() string { - return generationFileName +// GetMetricsTempFileName gets the file name used to temporarily store new metrics. +func GetMetricsTempFileName() string { + return metricsTempFileName } -// getGenerationNumber opens the generation.txt file and parses the payload into -// an integer. This integer represents the current generation of the metrics file. -func getGenerationNumber(filePath string) (int, error) { - genFile, err := os.Open(filePath) +// getGenerationNumber lists the files in the directory and matches all files +// that follow the metricsFileRegex format. We then extract the generation number +// from all the filenames and return the highest/latest generation number. +func getGenerationNumber(dirPath string) (int, error) { + pattern := regexp.MustCompile(metricsFileRegex) + dirContents, err := os.ReadDir(dirPath) if err != nil { - return -1, fmt.Errorf("failed to open generation file %q: %w", filePath, err) + return -1, fmt.Errorf(`failed to list items in directory "%q": %w`, dirPath, err) } - defer genFile.Close() - // Create file reader - reader := bufio.NewReader(genFile) + highestGeneration := 0 + for _, item := range dirContents { + if item.IsDir() { + continue + } - // Read the first line. - input, _ := reader.ReadString('\n') - input = strings.TrimSpace(input) // Remove leading/trailing whitespace + matches := pattern.FindStringSubmatch(item.Name()) + if len(matches) > 1 { + x, err := strconv.Atoi(matches[1]) + if err != nil { + return -1, fmt.Errorf("failed to convert generation number in metrics file name %q: %w", matches[1], err) + } + highestGeneration = max(highestGeneration, x) + } + } - // Convert to integer - num, err := strconv.Atoi(input) - if err != nil { - return -1, fmt.Errorf(`invalid input "%s" must be an integer: %w`, input, err) + if highestGeneration == 0 { + return -1, errors.New("failed to get latest generation: directory does not contain any metric files") } - return num, nil + return highestGeneration, nil } // emitMetricFamily iterates MetricFamily, converts metricFamily.Metric to prometheus.Metric, and emits the metric via the given chan. diff --git a/pkg/sidecar_mounter/logger.go b/pkg/sidecar_mounter/logger.go index 44fac5067..b20b9bdd6 100644 --- a/pkg/sidecar_mounter/logger.go +++ b/pkg/sidecar_mounter/logger.go @@ -57,7 +57,7 @@ func (w *stderrWriter) Write(msg []byte) (int, error) { // WriteMsg calls Write func and handles errors. func (w *stderrWriter) WriteMsg(errMsg string) { - klog.Errorf(errMsg) + klog.Error(errMsg) if _, e := w.Write([]byte(errMsg)); e != nil { klog.Errorf("failed to write the error message %q: %v", errMsg, e) } diff --git a/pkg/sidecar_mounter/sidecar_mounter.go b/pkg/sidecar_mounter/sidecar_mounter.go index 9d61db9ef..fe29f0b14 100644 --- a/pkg/sidecar_mounter/sidecar_mounter.go +++ b/pkg/sidecar_mounter/sidecar_mounter.go @@ -49,7 +49,7 @@ func New(mounterPath string) *Mounter { } } -func (m *Mounter) Mount(ctx context.Context, mc *MountConfig) error { +func (m *Mounter) Mount(ctx context.Context, mc *MountConfig, metricsScrapeInterval int) error { klog.Infof("start to mount bucket %q for volume %q", mc.BucketName, mc.VolumeName) if err := os.MkdirAll(mc.BufferDir+TempDir, os.ModePerm); err != nil { @@ -114,8 +114,8 @@ func (m *Mounter) Mount(ctx context.Context, mc *MountConfig) error { promPort := mc.FlagMap["prometheus-port"] if promPort != "0" { - klog.Infof("start to collect metrics from port %v for volume %q", promPort, mc.VolumeName) - go collectMetrics(ctx, promPort, mc.TempDir) + klog.Infof("start to collect metrics from port %v for volume %q with scrape interval of %d second(s)", promPort, mc.VolumeName, metricsScrapeInterval) + go collectMetrics(ctx, promPort, mc.TempDir, metricsScrapeInterval) } // Since the gcsfuse has taken over the file descriptor, @@ -214,10 +214,10 @@ func logVolumeTotalSize(dirPath string) { } // collectMetrics collects metrics from the gcsfuse instance every 10 seconds. -func collectMetrics(ctx context.Context, port, dirPath string) { - genNumber := 0 +func collectMetrics(ctx context.Context, port, dirPath string, scrapeInterval int) { + genNumber := 1 metricEndpoint := "http://localhost:" + port + "/metrics" - ticker := time.NewTicker(10 * time.Second) + ticker := time.NewTicker(time.Duration(scrapeInterval) * time.Second) for { select { @@ -226,6 +226,9 @@ func collectMetrics(ctx context.Context, port, dirPath string) { case <-ticker.C: newCtx, cancel := context.WithTimeout(ctx, 5*time.Second) scrapeMetrics(newCtx, metricEndpoint, dirPath, genNumber) + // If x is the current generation of the metrics snapshot/file, we are leaving some buffer where the outdated (x-1) + // file exists so that the reader has time to finish reading the complete data. + // The -2 means we are deleting file generations up to x - 2. deleteOutdatedFile(dirPath, genNumber-2) cancel() genNumber++ @@ -233,9 +236,11 @@ func collectMetrics(ctx context.Context, port, dirPath string) { } } -// scrapeMetrics connects to the metrics endpoint, scrapes metrics, and save the metrics to the given file path. +// scrapeMetrics connects to the metrics endpoint and scrapes latest metrics sample. +// We write the metrics to a temporary file while we finalize scrape, and then rename +// the file to match format of metrics_x.prom, where x is the latest sample/generation. func scrapeMetrics(ctx context.Context, metricEndpoint, dirPath string, genNumber int) { - outputPath := dirPath + metrics.GetMetricsFileName(genNumber) + tempPath := filepath.Join(dirPath, metrics.GetMetricsTempFileName()) // Make the HTTP GET request req, err := http.NewRequestWithContext(ctx, http.MethodGet, metricEndpoint, nil) @@ -261,7 +266,7 @@ func scrapeMetrics(ctx context.Context, metricEndpoint, dirPath string, genNumbe } // Create the output file - out, err := os.Create(outputPath) + out, err := os.Create(tempPath) if err != nil { klog.Errorf("error creating output file: %v", err) @@ -279,31 +284,25 @@ func scrapeMetrics(ctx context.Context, metricEndpoint, dirPath string, genNumbe // Ensure closure of output file. out.Close() - // Update generation number. - generationFilePath := dirPath + metrics.GetGenerationFileName() - genFile, err := os.Create(generationFilePath) - if err != nil { - klog.Errorf("error getting file descriptor for metrics generation file: %v", err) - - return - } - defer genFile.Close() + // Get final filename. + outputPath := filepath.Join(dirPath, metrics.GetMetricsFileName(genNumber)) - _, err = fmt.Fprintf(genFile, "%d\n", genNumber) + // Update file name with metrics, atomic operation. + err = os.Rename(tempPath, outputPath) if err != nil { - klog.Infof("Error writing to file: %v", err) + klog.Errorf("error renaming metrics file from temp to %s format: %v", outputPath, err) return } } func deleteOutdatedFile(dirPath string, genNumber int) { - if genNumber < 0 { + if genNumber < 1 { return } // Create path for file to delete. - outputPath := dirPath + metrics.GetMetricsFileName(genNumber) + outputPath := filepath.Join(dirPath, metrics.GetMetricsFileName(genNumber)) // Delete the output file. err := os.Remove(outputPath) diff --git a/pkg/sidecar_mounter/sidecar_mounter_config.go b/pkg/sidecar_mounter/sidecar_mounter_config.go index f1aa8075e..28c5a8803 100644 --- a/pkg/sidecar_mounter/sidecar_mounter_config.go +++ b/pkg/sidecar_mounter/sidecar_mounter_config.go @@ -158,6 +158,7 @@ func (mc *MountConfig) prepareMountArgs() { "gid": "0", "prometheus-port": strconv.Itoa(prometheusPort), } + // Use a new port each gcsfuse instance that we start. prometheusPort++ configFileFlagMap := map[string]string{ diff --git a/pkg/webhook/validating_admission_policy_test.go b/pkg/webhook/validating_admission_policy_test.go new file mode 100644 index 000000000..11807af70 --- /dev/null +++ b/pkg/webhook/validating_admission_policy_test.go @@ -0,0 +1,268 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "bytes" + "context" + "os" + "testing" + + "github.com/google/go-cmp/cmp" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/admission/plugin/cel" + "k8s.io/apiserver/pkg/admission/plugin/policy/validating" + "k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions" + celconfig "k8s.io/apiserver/pkg/apis/cel" + "k8s.io/apiserver/pkg/cel/environment" + "k8s.io/utils/ptr" +) + +func testPod(initContainer bool, annotation map[string]string, restartPolicy *corev1.ContainerRestartPolicy, env []corev1.EnvVar) *corev1.Pod { + container := corev1.Container{ + Name: SidecarContainerName, + RestartPolicy: restartPolicy, + Env: env, + } + + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotation, + }, + Spec: corev1.PodSpec{}, + } + + if initContainer { + pod.Spec.InitContainers = []corev1.Container{container} + } else { + pod.Spec.Containers = []corev1.Container{container} + } + + return &pod +} + +func expectedValidateResult(d1, d2 *validating.PolicyDecision) validating.ValidateResult { + vr := validating.ValidateResult{ + Decisions: []validating.PolicyDecision{}, + AuditAnnotations: []validating.PolicyAuditAnnotation{}, + } + + if d1 != nil { + vr.Decisions = append(vr.Decisions, *d1) + } + + if d2 != nil { + vr.Decisions = append(vr.Decisions, *d2) + } + + if d1 == nil && d2 == nil { + vr.Decisions = nil + vr.AuditAnnotations = nil + } + + return vr +} + +var admittedDecision = validating.PolicyDecision{ + Action: validating.ActionAdmit, + Evaluation: validating.EvalAdmit, +} + +var missingRestartPolicyDecision = validating.PolicyDecision{ + Action: validating.ActionDeny, + Message: "the native gcsfuse sidecar init container must have restartPolicy:Always.", + Reason: metav1.StatusReasonInvalid, +} + +var missingEnvVarDecision = validating.PolicyDecision{ + Action: validating.ActionDeny, + Message: "the native gcsfuse sidecar init container must have env var NATIVE_SIDECAR with value TRUE.", + Reason: metav1.StatusReasonInvalid, +} + +var testCases = []struct { + name string + pod *corev1.Pod + expectedResult validating.ValidateResult +}{ + { + name: "validation passed with a valid native sidecar container", + pod: testPod(true, map[string]string{GcsFuseVolumeEnableAnnotation: "true"}, ptr.To(corev1.ContainerRestartPolicyAlways), []corev1.EnvVar{{Name: "NATIVE_SIDECAR", Value: "TRUE"}}), + expectedResult: expectedValidateResult(&admittedDecision, &admittedDecision), + }, + { + name: "validation failed with a native sidecar container missing RestartPolicy", + pod: testPod(true, map[string]string{GcsFuseVolumeEnableAnnotation: "true"}, nil, []corev1.EnvVar{{Name: "NATIVE_SIDECAR", Value: "TRUE"}}), + expectedResult: expectedValidateResult(&missingRestartPolicyDecision, &admittedDecision), + }, + { + name: "validation failed with a native sidecar container missing env var", + pod: testPod(true, map[string]string{GcsFuseVolumeEnableAnnotation: "true"}, ptr.To(corev1.ContainerRestartPolicyAlways), nil), + expectedResult: expectedValidateResult(&admittedDecision, &missingEnvVarDecision), + }, + { + name: "validation failed with a valid native sidecar container missing correct env var", + pod: testPod(true, map[string]string{GcsFuseVolumeEnableAnnotation: "true"}, ptr.To(corev1.ContainerRestartPolicyAlways), []corev1.EnvVar{{Name: "NATIVE_SIDECAR", Value: "FALSE"}}), + expectedResult: expectedValidateResult(&admittedDecision, &missingEnvVarDecision), + }, + { + name: "validation failed with a native sidecar container missing RestartPolicy and env var", + pod: testPod(true, map[string]string{GcsFuseVolumeEnableAnnotation: "true"}, nil, nil), + expectedResult: expectedValidateResult(&missingRestartPolicyDecision, &missingEnvVarDecision), + }, + { + name: "validation skipped without pod annotations", + pod: testPod(true, nil, nil, nil), + expectedResult: expectedValidateResult(nil, nil), + }, + { + name: "validation skipped with a pod annotation gke-gcsfuse/volumes: false", + pod: testPod(true, map[string]string{GcsFuseVolumeEnableAnnotation: "false"}, nil, nil), + expectedResult: expectedValidateResult(nil, nil), + }, + { + name: "validation skipped with a regular sidecar container", + pod: testPod(false, map[string]string{GcsFuseVolumeEnableAnnotation: "true"}, nil, nil), + expectedResult: expectedValidateResult(nil, nil), + }, +} + +func TestValidatingAdmissionPolicy(t *testing.T) { + t.Parallel() + + filename := "../../deploy/base/webhook/validating_admission_policy.yaml" + policy, err := loadValidatingAdmissionPolicy(filename) + if err != nil { + t.Fatalf("failed to load policy: %v", err) + } + + validator := compilePolicy(policy) + + for _, tc := range testCases { + fakeAttr := admission.NewAttributesRecord(tc.pod, nil, schema.GroupVersionKind{}, "", "", schema.GroupVersionResource{}, "", "", nil, false, nil) + fakeVersionedAttr, _ := admission.NewVersionedAttributes(fakeAttr, schema.GroupVersionKind{}, nil) + validateResult := validator.Validate(context.TODO(), fakeVersionedAttr.GetResource(), fakeVersionedAttr, nil, nil, celconfig.RuntimeCELCostBudget, nil) + + if diff := cmp.Diff(validateResult, tc.expectedResult); diff != "" { + t.Errorf("unexpected options args (-got, +want)\n%s", diff) + } + } +} + +func loadValidatingAdmissionPolicy(filename string) (*admissionregistrationv1.ValidatingAdmissionPolicy, error) { + data, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + decoder := yaml.NewYAMLToJSONDecoder(bytes.NewReader(data)) + + obj := &admissionregistrationv1.ValidatingAdmissionPolicy{} + err = decoder.Decode(obj) + + return obj, err +} + +func compilePolicy(policy *admissionregistrationv1.ValidatingAdmissionPolicy) validating.Validator { + hasParam := false + if policy.Spec.ParamKind != nil { + hasParam = true + } + optionalVars := cel.OptionalVariableDeclarations{HasParams: hasParam, HasAuthorizer: true, StrictCost: true} + expressionOptionalVars := cel.OptionalVariableDeclarations{HasParams: hasParam, HasAuthorizer: false, StrictCost: true} + failurePolicy := policy.Spec.FailurePolicy + matchConditions := policy.Spec.MatchConditions + + compositionEnvTemplate, err := cel.NewCompositionEnv(cel.VariablesTypeName, environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion(), true)) + if err != nil { + panic(err) + } + filterCompiler := cel.NewCompositedCompilerFromTemplate(compositionEnvTemplate) + filterCompiler.CompileAndStoreVariables(convertv1beta1Variables(policy.Spec.Variables), optionalVars, environment.StoredExpressions) + + matchExpressionAccessors := make([]cel.ExpressionAccessor, len(matchConditions)) + for i := range matchConditions { + matchExpressionAccessors[i] = (*matchconditions.MatchCondition)(&matchConditions[i]) + } + + matcher := matchconditions.NewMatcher(filterCompiler.Compile(matchExpressionAccessors, optionalVars, environment.StoredExpressions), failurePolicy, "policy", "validate", policy.Name) + res := validating.NewValidator( + filterCompiler.Compile(convertv1Validations(policy.Spec.Validations), optionalVars, environment.StoredExpressions), + matcher, + filterCompiler.Compile(convertv1AuditAnnotations(policy.Spec.AuditAnnotations), optionalVars, environment.StoredExpressions), + filterCompiler.Compile(convertv1MessageExpressions(policy.Spec.Validations), expressionOptionalVars, environment.StoredExpressions), + failurePolicy, + ) + + return res +} + +func convertv1Validations(inputValidations []admissionregistrationv1.Validation) []cel.ExpressionAccessor { + celExpressionAccessor := make([]cel.ExpressionAccessor, len(inputValidations)) + for i, validation := range inputValidations { + validation := validating.ValidationCondition{ + Expression: validation.Expression, + Message: validation.Message, + Reason: validation.Reason, + } + celExpressionAccessor[i] = &validation + } + + return celExpressionAccessor +} + +func convertv1MessageExpressions(inputValidations []admissionregistrationv1.Validation) []cel.ExpressionAccessor { + celExpressionAccessor := make([]cel.ExpressionAccessor, len(inputValidations)) + for i, validation := range inputValidations { + if validation.MessageExpression != "" { + condition := validating.MessageExpressionCondition{ + MessageExpression: validation.MessageExpression, + } + celExpressionAccessor[i] = &condition + } + } + + return celExpressionAccessor +} + +func convertv1AuditAnnotations(inputValidations []admissionregistrationv1.AuditAnnotation) []cel.ExpressionAccessor { + celExpressionAccessor := make([]cel.ExpressionAccessor, len(inputValidations)) + for i, validation := range inputValidations { + validation := validating.AuditAnnotationCondition{ + Key: validation.Key, + ValueExpression: validation.ValueExpression, + } + celExpressionAccessor[i] = &validation + } + + return celExpressionAccessor +} + +func convertv1beta1Variables(variables []admissionregistrationv1.Variable) []cel.NamedExpressionAccessor { + namedExpressions := make([]cel.NamedExpressionAccessor, len(variables)) + for i, variable := range variables { + namedExpressions[i] = &validating.Variable{Name: variable.Name, Expression: variable.Expression} + } + + return namedExpressions +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 75cf7dfd5..8168c6576 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -108,6 +108,7 @@ var _ = ginkgo.Describe("E2E Test Suite", func() { testsuites.InitGcsFuseCSIAutoTerminationTestSuite, testsuites.InitGcsFuseCSIFileCacheTestSuite, testsuites.InitGcsFuseCSIGCSFuseIntegrationFileCacheTestSuite, + testsuites.InitGcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite, testsuites.InitGcsFuseCSIIstioTestSuite, testsuites.InitGcsFuseCSIMetricsTestSuite, } diff --git a/test/e2e/main.go b/test/e2e/main.go index ed5bf4dd9..8333c5595 100644 --- a/test/e2e/main.go +++ b/test/e2e/main.go @@ -38,6 +38,7 @@ var ( useGKEAutopilot = flag.Bool("use-gke-autopilot", false, "use GKE Autopilot cluster for the tests") apiEndpointOverride = flag.String("api-endpoint-override", "https://container.googleapis.com/", "CloudSDK API endpoint override to use for the cluster environment") nodeImageType = flag.String("node-image-type", "cos_containerd", "image type to use for the cluster") + istioVersion = flag.String("istio-version", "1.23.0", "istio version to install on the cluster") // Test infrastructure flags. inProw = flag.Bool("run-in-prow", false, "whether or not to run the test in PROW") @@ -101,6 +102,7 @@ func main() { GinkgoTimeout: *ginkgoTimeout, GinkgoFlakeAttempts: *ginkgoFlakeAttempts, GinkgoSkipGcpSaTest: *ginkgoSkipGcpSaTest, + IstioVersion: *istioVersion, } if strings.Contains(testParams.GinkgoFocus, "performance") { diff --git a/test/e2e/run-e2e-ci.sh b/test/e2e/run-e2e-ci.sh index 97f7c7588..dac9323ba 100755 --- a/test/e2e/run-e2e-ci.sh +++ b/test/e2e/run-e2e-ci.sh @@ -37,7 +37,7 @@ readonly node_machine_type=${MACHINE_TYPE:-n2-standard-4} readonly number_nodes=${NUMBER_NODES:-1} # Install golang -version=1.22.3 +version=1.23.0 wget -O go_tar.tar.gz https://go.dev/dl/go${version}.linux-amd64.tar.gz -q rm -rf /usr/local/go && tar -xzf go_tar.tar.gz -C /usr/local export PATH=$PATH:/usr/local/go/bin && go version && rm go_tar.tar.gz diff --git a/test/e2e/specs/istio-service-entry.yaml b/test/e2e/specs/istio-service-entry.yaml new file mode 100644 index 000000000..c7f166ec3 --- /dev/null +++ b/test/e2e/specs/istio-service-entry.yaml @@ -0,0 +1,30 @@ +# Copyright 2018 The Kubernetes Authors. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: networking.istio.io/v1 +kind: ServiceEntry +metadata: + name: googleapi +spec: + hosts: + - storage.googleapis.com + exportTo: + - "." # restricts the visibility to the current namespace + location: MESH_EXTERNAL + ports: + - name: https + number: 443 + protocol: TLS + resolution: DNS diff --git a/test/e2e/specs/istio-sidecar.yaml b/test/e2e/specs/istio-sidecar.yaml new file mode 100644 index 000000000..10fd5544b --- /dev/null +++ b/test/e2e/specs/istio-sidecar.yaml @@ -0,0 +1,22 @@ +# Copyright 2018 The Kubernetes Authors. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: networking.istio.io/v1 +kind: Sidecar +metadata: + name: default +spec: + outboundTrafficPolicy: + mode: REGISTRY_ONLY diff --git a/test/e2e/specs/specs.go b/test/e2e/specs/specs.go index 57eff1837..d6bfaeef9 100644 --- a/test/e2e/specs/specs.go +++ b/test/e2e/specs/specs.go @@ -41,6 +41,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2ejob "k8s.io/kubernetes/test/e2e/framework/job" + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epodooutput "k8s.io/kubernetes/test/e2e/framework/pod/output" storageframework "k8s.io/kubernetes/test/e2e/storage/framework" @@ -73,7 +74,7 @@ const ( SkipCSIBucketAccessCheckAndImplicitDirsVolumePrefix = "gcsfuse-csi-skip-bucket-access-check-implicit-dirs-volume" GoogleCloudCliImage = "gcr.io/google.com/cloudsdktool/google-cloud-cli:slim" - GolangImage = "golang:1.22.1" + GolangImage = "golang:1.23.0" UbuntuImage = "ubuntu:20.04" LastPublishedSidecarContainerImage = "gcr.io/gke-release/gcs-fuse-csi-driver-sidecar-mounter@sha256:c83609ecf50d05a141167b8c6cf4dfe14ff07f01cd96a9790921db6748d40902" @@ -109,6 +110,7 @@ func NewTestPod(c clientset.Interface, ns *corev1.Namespace) *TestPod { "gke-gcsfuse/memory-request": "100Mi", "gke-gcsfuse/ephemeral-storage-request": "100Mi", }, + Labels: map[string]string{}, }, Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: ptr.To(int64(5)), @@ -222,6 +224,17 @@ func (t *TestPod) WaitForFailedMountError(ctx context.Context, msg string) { framework.ExpectNoError(err) } +func (t *TestPod) WaitForFailedContainerError(ctx context.Context, msg string) { + err := e2eevents.WaitTimeoutForEvent( + ctx, + t.client, + t.namespace.Name, + fields.Set{"reason": events.FailedToStartContainer}.AsSelector().String(), + msg, + pollTimeoutSlow) + framework.ExpectNoError(err) +} + func (t *TestPod) WaitForPodNotFoundInNamespace(ctx context.Context) { err := e2epod.WaitForPodNotFoundInNamespace(ctx, t.client, t.pod.Name, t.namespace.Name, pollTimeout) framework.ExpectNoError(err) @@ -348,7 +361,7 @@ func (t *TestPod) GetNode() string { return t.pod.Spec.NodeName } -func (t *TestPod) GetCISDriverNodePodIP(ctx context.Context) string { +func (t *TestPod) GetCSIDriverNodePodIP(ctx context.Context) string { node := t.GetNode() daemonSetLabelSelector := "k8s-app=gcs-fuse-csi-driver" @@ -387,6 +400,12 @@ func (t *TestPod) SetAnnotations(annotations map[string]string) { } } +func (t *TestPod) SetLabels(labels map[string]string) { + for k, v := range labels { + t.pod.Labels[k] = v + } +} + func (t *TestPod) SetServiceAccount(sa string) { t.pod.Spec.ServiceAccountName = sa } @@ -411,55 +430,6 @@ func (t *TestPod) SetCommand(cmd string) { t.pod.Spec.Containers[0].Args = []string{"-c", cmd} } -func (t *TestPod) SetIstioSidecar(isNativeSidecar bool) { - istioSidecar := []corev1.Container{{ - Name: webhook.IstioSidecarName, - Image: imageutils.GetE2EImage(imageutils.BusyBox), - }} - if isNativeSidecar { - t.pod.Spec.InitContainers = append(istioSidecar, t.pod.Spec.InitContainers...) - } else { - t.pod.Spec.Containers = append(istioSidecar, t.pod.Spec.Containers...) - } -} - -func (t *TestPod) VerifyInjectionOrder(gcsFuseNativeSidecar, istioNativeSidecar bool) { - if gcsFuseNativeSidecar { - // Expect the gcsfuse sidecar to be native. - if istioNativeSidecar { - // Expect both gcsfuse and istio to be native sidecar. - gomega.Expect(t.pod.Spec.InitContainers).To(gomega.HaveLen(2)) - - // Verify ordering of sidecars. - gomega.Expect(t.pod.Spec.InitContainers[0].Name).To(gomega.BeEquivalentTo(webhook.IstioSidecarName)) - gomega.Expect(t.pod.Spec.InitContainers[1].Name).To(gomega.BeEquivalentTo(webhook.SidecarContainerName)) - - // Verify workload is present - gomega.Expect(t.pod.Spec.Containers).To(gomega.HaveLen(1)) - } else { - gomega.Panic().FailureMessage("container istio-proxy is not supported with native GCSFuse sidecar") - } - } else { - // Expect gcsfuse sidecar to be regular container. - if istioNativeSidecar { - // Verify istio-proxy is first in native containers. - gomega.Expect(t.pod.Spec.InitContainers).To(gomega.HaveLen(1)) - gomega.Expect(t.pod.Spec.InitContainers[0].Name).To(gomega.BeEquivalentTo(webhook.IstioSidecarName)) - - // Verify gcsfuse is first in regular containers. - gomega.Expect(t.pod.Spec.Containers).To(gomega.HaveLen(2)) - gomega.Expect(t.pod.Spec.Containers[0].Name).To(gomega.BeEquivalentTo(webhook.SidecarContainerName)) - } else { - // Expect both gcsfuse and istio to be regular containers. - gomega.Expect(t.pod.Spec.Containers).To(gomega.HaveLen(3)) - - // Verify ordering of sidecars. - gomega.Expect(t.pod.Spec.Containers[0].Name).To(gomega.BeEquivalentTo(webhook.IstioSidecarName)) - gomega.Expect(t.pod.Spec.Containers[1].Name).To(gomega.BeEquivalentTo(webhook.SidecarContainerName)) - } - } -} - func (t *TestPod) SetGracePeriod(s int) { t.pod.Spec.TerminationGracePeriodSeconds = ptr.To(int64(s)) } @@ -1183,8 +1153,17 @@ func GetGCSFuseVersion(ctx context.Context, client clientset.Interface) string { "/gcsfuse --version should succeed, but failed with error message %q\nstdout: %s\nstderr: %s", err, stdout, stderr) - l := strings.Split(stderr, " ") + // Before GCSFuse v2.4.1, the output of is saved in stderr + l := strings.Split(stdout+stderr, " ") gomega.Expect(len(l)).To(gomega.BeNumerically(">", 3)) return l[2] } + +func DeployIstioSidecar(namespace string) { + e2ekubectl.RunKubectlOrDie(namespace, "apply", "--filename", "./specs/istio-sidecar.yaml") +} + +func DeployIstioServiceEntry(namespace string) { + e2ekubectl.RunKubectlOrDie(namespace, "apply", "--filename", "./specs/istio-service-entry.yaml") +} diff --git a/test/e2e/testsuites/failed_mount.go b/test/e2e/testsuites/failed_mount.go index 24c5060f7..b6250f603 100644 --- a/test/e2e/testsuites/failed_mount.go +++ b/test/e2e/testsuites/failed_mount.go @@ -314,7 +314,6 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes }) testcaseInvalidMountOptions := func(configPrefix string) { - // init(specs.InvalidMountOptionsVolumePrefix) init(configPrefix) defer cleanup() @@ -328,7 +327,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Checking that the pod has failed mount error") tPod.WaitForFailedMountError(ctx, codes.InvalidArgument.String()) - tPod.WaitForFailedMountError(ctx, "Incorrect Usage. flag provided but not defined: -invalid-option") + tPod.WaitForFailedMountError(ctx, "-invalid-option") } ginkgo.It("should fail when invalid mount options are passed", func() { diff --git a/test/e2e/testsuites/gcsfuse_integration.go b/test/e2e/testsuites/gcsfuse_integration.go index e35eb48ed..3942886af 100644 --- a/test/e2e/testsuites/gcsfuse_integration.go +++ b/test/e2e/testsuites/gcsfuse_integration.go @@ -51,6 +51,8 @@ const ( testNameKernelListCache = "kernel_list_cache" testNamePrefixSucceed = "should succeed in " + + masterBranchName = "master" ) var gcsfuseVersionStr = "" @@ -113,7 +115,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew // the version format does not obey the syntax and semantics of the "Semantic Versioning". // Always use master branch if the gcsfuse binary is built using the head commit. if err != nil { - return "master" + return masterBranchName } // check if the given gcsfuse version supports the test case @@ -206,26 +208,40 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, "apt-get update && apt-get install -y google-cloud-cli") ginkgo.By("Checking that the gcsfuse integration tests exits with no error") - baseTestCommand := fmt.Sprintf("export GOTOOLCHAIN=go1.22.4 && export PATH=$PATH:/usr/local/go/bin && cd %v/%v && GODEBUG=asyncpreemptoff=1 go test . -p 1 --integrationTest -v --mountedDirectory=%v", gcsfuseIntegrationTestsBasePath, testName, mountPath) + baseTestCommand := fmt.Sprintf("export GOTOOLCHAIN=go1.23.0 && export PATH=$PATH:/usr/local/go/bin && cd %v/%v && GODEBUG=asyncpreemptoff=1 go test . -p 1 --integrationTest -v --mountedDirectory=%v", gcsfuseIntegrationTestsBasePath, testName, mountPath) baseTestCommandWithTestBucket := baseTestCommand + fmt.Sprintf(" --testbucket=%v", bucketName) + + var finalTestCommand string switch testName { case testNameReadonly: if readOnly { - tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, baseTestCommandWithTestBucket) + finalTestCommand = baseTestCommandWithTestBucket } else { - tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, fmt.Sprintf("chmod 777 %v/readonly && useradd -u 6666 -m test-user && su test-user -c '%v'", gcsfuseIntegrationTestsBasePath, baseTestCommandWithTestBucket)) + finalTestCommand = fmt.Sprintf("chmod 777 %v/readonly && useradd -u 6666 -m test-user && su test-user -c '%v'", gcsfuseIntegrationTestsBasePath, baseTestCommandWithTestBucket) } case testNameExplicitDir, testNameImplicitDir, testNameGzip, testNameLocalFile, testNameOperations, testNameConcurrentOperations: - tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, baseTestCommandWithTestBucket) + finalTestCommand = baseTestCommandWithTestBucket + case testNameRenameDirLimit: + if gcsfuseTestBranch == masterBranchName || version.MustParseSemantic(gcsfuseTestBranch).AtLeast(version.MustParseSemantic("v2.4.1")) { + finalTestCommand = baseTestCommandWithTestBucket + } else { + finalTestCommand = baseTestCommand + } case testNameKernelListCache, testNameManagedFolders: - tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, baseTestCommandWithTestBucket+" -run "+testCase) + finalTestCommand = baseTestCommandWithTestBucket + " -run " + testCase case testNameListLargeDir, testNameWriteLargeFiles: - tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, baseTestCommandWithTestBucket+" -timeout 80m") + finalTestCommand = baseTestCommandWithTestBucket + " -timeout 80m" case testNameReadLargeFiles: - tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, baseTestCommand+" -timeout 60m") + if gcsfuseTestBranch == masterBranchName || version.MustParseSemantic(gcsfuseTestBranch).AtLeast(version.MustParseSemantic("v2.4.1")) { + finalTestCommand = baseTestCommandWithTestBucket + " -timeout 60m" + } else { + finalTestCommand = baseTestCommand + " -timeout 60m" + } default: - tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, baseTestCommand) + finalTestCommand = baseTestCommand } + + tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, finalTestCommand) } testNameSuffix := func(i int) string { @@ -463,13 +479,27 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew init() defer cleanup() - gcsfuseIntegrationTest(testNameKernelListCache+":TestFiniteKernelListCacheTest/TestKernelListCache_CacheHitWithinLimit_CacheMissAfterLimit", false, "implicit-dirs=true", "kernel-list-cache-ttl-secs=5") + gcsfuseIntegrationTest(testNameKernelListCache+":TestInfiniteKernelListCacheTest/TestKernelListCache_ListAndDeleteDirectory", false, "implicit-dirs=true", "kernel-list-cache-ttl-secs=-1") }) ginkgo.It(testNamePrefixSucceed+testNameKernelListCache+testNameSuffix(10), func() { init() defer cleanup() + gcsfuseIntegrationTest(testNameKernelListCache+":TestInfiniteKernelListCacheTest/TestKernelListCache_DeleteAndListDirectory", false, "implicit-dirs=true", "kernel-list-cache-ttl-secs=-1") + }) + + ginkgo.It(testNamePrefixSucceed+testNameKernelListCache+testNameSuffix(11), func() { + init() + defer cleanup() + + gcsfuseIntegrationTest(testNameKernelListCache+":TestFiniteKernelListCacheTest/TestKernelListCache_CacheHitWithinLimit_CacheMissAfterLimit", false, "implicit-dirs=true", "kernel-list-cache-ttl-secs=5") + }) + + ginkgo.It(testNamePrefixSucceed+testNameKernelListCache+testNameSuffix(12), func() { + init() + defer cleanup() + gcsfuseIntegrationTest(testNameKernelListCache+":TestDisabledKernelListCacheTest/TestKernelListCache_AlwaysCacheMiss", false, "implicit-dirs=true", "kernel-list-cache-ttl-secs=0") }) diff --git a/test/e2e/testsuites/gcsfuse_integration_file_cache.go b/test/e2e/testsuites/gcsfuse_integration_file_cache.go index 8334112bc..9f5d886ac 100644 --- a/test/e2e/testsuites/gcsfuse_integration_file_cache.go +++ b/test/e2e/testsuites/gcsfuse_integration_file_cache.go @@ -91,7 +91,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheTestSuite) DefineTests(driver stor // the version format does not obey the syntax and semantics of the "Semantic Versioning". // Always use master branch if the gcsfuse binary is built using the head commit. if err != nil { - return "master" + return masterBranchName } // check if the given gcsfuse version supports the test case @@ -126,7 +126,11 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheTestSuite) DefineTests(driver stor l.volumeResource.VolSource.CSI.VolumeAttributes["metadataCacheTTLSeconds"] = metadataCacheTTLSeconds tPod.SetupTmpVolumeMount("/tmp/gcsfuse_read_cache_test_logs") - tPod.SetupCacheVolumeMount("/tmp/cache-dir", ".volumes/"+volumeName) + cacheDir := "cache-dir" + if gcsfuseTestBranch == masterBranchName || version.MustParseSemantic(gcsfuseTestBranch).AtLeast(version.MustParseSemantic("v2.4.1")) { + cacheDir = "cache-dir-read-cache-hns-false" + } + tPod.SetupCacheVolumeMount("/tmp/"+cacheDir, ".volumes/"+volumeName) mountOptions = append(mountOptions, "logging:file-path:/gcsfuse-tmp/log.json", "logging:format:json", "logging:severity:trace") tPod.SetupVolume(l.volumeResource, volumeName, mountPath, readOnly, mountOptions...) @@ -155,7 +159,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheTestSuite) DefineTests(driver stor ginkgo.By("Checking that the gcsfuse integration tests exits with no error") tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("git clone --branch %v https://github.com/GoogleCloudPlatform/gcsfuse.git", gcsfuseTestBranch)) - baseTestCommand := fmt.Sprintf("export GOTOOLCHAIN=go1.22.4 && export PATH=$PATH:/usr/local/go/bin && cd %v/read_cache && GODEBUG=asyncpreemptoff=1 go test . -p 1 --integrationTest -v --mountedDirectory=%v --testbucket=%v -run %v", gcsfuseIntegrationTestsBasePath, mountPath, bucketName, testName) + baseTestCommand := fmt.Sprintf("export GOTOOLCHAIN=go1.23.0 && export PATH=$PATH:/usr/local/go/bin && cd %v/read_cache && GODEBUG=asyncpreemptoff=1 go test . -p 1 --integrationTest -v --mountedDirectory=%v --testbucket=%v -run %v", gcsfuseIntegrationTestsBasePath, mountPath, bucketName, testName) tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, baseTestCommand) } @@ -214,30 +218,16 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheTestSuite) DefineTests(driver stor init() defer cleanup() - gcsfuseIntegrationFileCacheTest("TestRangeReadTest/TestRangeReadsBeyondReadChunkSizeWithoutChunkDownloaded", false, "500Mi", "false", "3600") - }) - - ginkgo.It("should succeed in TestRangeReadTest 4", func() { - init() - defer cleanup() - gcsfuseIntegrationFileCacheTest("TestRangeReadTest/TestRangeReadsWithinReadChunkSize", false, "500Mi", "true", "3600") }) - ginkgo.It("should succeed in TestRangeReadTest 5", func() { + ginkgo.It("should succeed in TestRangeReadTest 4", func() { init() defer cleanup() gcsfuseIntegrationFileCacheTest("TestRangeReadTest/TestRangeReadsBeyondReadChunkSizeWithChunkDownloaded", false, "500Mi", "true", "3600") }) - ginkgo.It("should succeed in TestRangeReadTest 6", func() { - init() - defer cleanup() - - gcsfuseIntegrationFileCacheTest("TestRangeReadTest/TestRangeReadsBeyondReadChunkSizeWithoutChunkDownloaded", false, "500Mi", "true", "3600") - }) - ginkgo.It("should succeed in TestReadOnlyTest 1", func() { init() defer cleanup() diff --git a/test/e2e/testsuites/gcsfuse_integration_file_cache_parallel_downloads.go b/test/e2e/testsuites/gcsfuse_integration_file_cache_parallel_downloads.go new file mode 100644 index 000000000..7bd7c7748 --- /dev/null +++ b/test/e2e/testsuites/gcsfuse_integration_file_cache_parallel_downloads.go @@ -0,0 +1,306 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "context" + "fmt" + "strings" + + "github.com/googlecloudplatform/gcs-fuse-csi-driver/test/e2e/specs" + "github.com/onsi/ginkgo/v2" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/kubernetes/test/e2e/framework" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" + storageframework "k8s.io/kubernetes/test/e2e/storage/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +type gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite struct { + tsInfo storageframework.TestSuiteInfo +} + +// InitGcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite returns gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite that implements TestSuite interface. +func InitGcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite() storageframework.TestSuite { + return &gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite{ + tsInfo: storageframework.TestSuiteInfo{ + Name: "gcsfuseIntegrationFileCacheParallelDownloads", + TestPatterns: []storageframework.TestPattern{ + storageframework.DefaultFsCSIEphemeralVolume, + }, + }, + } +} + +func (t *gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo { + return t.tsInfo +} + +func (t *gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite) SkipUnsupportedTests(_ storageframework.TestDriver, _ storageframework.TestPattern) { +} + +func (t *gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { + type local struct { + config *storageframework.PerTestConfig + volumeResource *storageframework.VolumeResource + } + var l local + ctx := context.Background() + + // Beware that it also registers an AfterEach which renders f unusable. Any code using + // f must run inside an It or Context callback. + f := framework.NewFrameworkWithCustomTimeouts("gcsfuse-integration-file-cache-parallel", storageframework.GetDriverTimeouts(driver)) + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + init := func(configPrefix ...string) { + l = local{} + l.config = driver.PrepareTest(ctx, f) + if len(configPrefix) > 0 { + l.config.Prefix = configPrefix[0] + } + l.volumeResource = storageframework.CreateVolumeResource(ctx, driver, l.config, pattern, e2evolume.SizeRange{}) + } + + cleanup := func() { + var cleanUpErrs []error + cleanUpErrs = append(cleanUpErrs, l.volumeResource.CleanupResource(ctx)) + err := utilerrors.NewAggregate(cleanUpErrs) + framework.ExpectNoError(err, "while cleaning up") + } + + skipTestOrProceedWithBranch := func(gcsfuseVersionStr, testName string) string { + v, err := version.ParseSemantic(gcsfuseVersionStr) + // When the gcsfuse binary is built using the head commit in the test pipeline, + // the version format does not obey the syntax and semantics of the "Semantic Versioning". + // Always use master branch if the gcsfuse binary is built using the head commit. + if err != nil { + return masterBranchName + } + + // check if the given gcsfuse version supports the test case + if !v.AtLeast(version.MustParseSemantic("v2.4.0-gke.0")) { + e2eskipper.Skipf("skip gcsfuse integration test %v for gcsfuse version %v", testName, v.String()) + } + + // By default, use the test code in the same release tag branch + return fmt.Sprintf("v%v.%v.%v", v.Major(), v.Minor(), v.Patch()) + } + + gcsfuseIntegrationFileCacheTest := func(testName string, readOnly bool, fileCacheCapacity, fileCacheForRangeRead, metadataCacheTTLSeconds string, mountOptions ...string) { + ginkgo.By("Checking GCSFuse version and skip test if needed") + if gcsfuseVersionStr == "" { + gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx, f.ClientSet) + } + ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, gcsfuseVersionStr)) + gcsfuseTestBranch := skipTestOrProceedWithBranch(gcsfuseVersionStr, testName) + ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse branch %v", testName, gcsfuseTestBranch)) + + ginkgo.By("Configuring the test pod") + tPod := specs.NewTestPod(f.ClientSet, f.Namespace) + tPod.SetImage(specs.GolangImage) + tPod.SetCommand("tail -F /tmp/gcsfuse_read_cache_test_logs/log.json") + tPod.SetResource("1", "1Gi", "5Gi") + if strings.HasPrefix(testName, "TestRangeReadTest") { + tPod.SetResource("1", "2Gi", "5Gi") + } + + l.volumeResource.VolSource.CSI.VolumeAttributes["fileCacheCapacity"] = fileCacheCapacity + l.volumeResource.VolSource.CSI.VolumeAttributes["fileCacheForRangeRead"] = fileCacheForRangeRead + l.volumeResource.VolSource.CSI.VolumeAttributes["metadataCacheTTLSeconds"] = metadataCacheTTLSeconds + + tPod.SetupTmpVolumeMount("/tmp/gcsfuse_read_cache_test_logs") + cacheDir := "cache-dir" + if gcsfuseTestBranch == masterBranchName || version.MustParseSemantic(gcsfuseTestBranch).AtLeast(version.MustParseSemantic("v2.4.1")) { + cacheDir = "cache-dir-read-cache-hns-false" + } + tPod.SetupCacheVolumeMount("/tmp/"+cacheDir, ".volumes/"+volumeName) + mountOptions = append(mountOptions, "logging:file-path:/gcsfuse-tmp/log.json", "logging:format:json", "logging:severity:trace") + mountOptions = append(mountOptions, + "file-cache:enable-parallel-downloads:true", + "file-cache:parallel-downloads-per-file:4", + "file-cache:max-parallel-downloads:-1", + "file-cache:download-chunk-size-mb:3", + "file-cache:enable-crc:true") + + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, readOnly, mountOptions...) + tPod.SetAnnotations(map[string]string{ + "gke-gcsfuse/cpu-limit": "250m", + "gke-gcsfuse/memory-limit": "256Mi", + "gke-gcsfuse/ephemeral-storage-limit": "2Gi", + }) + + bucketName := l.volumeResource.VolSource.CSI.VolumeAttributes["bucketName"] + + ginkgo.By("Deploying the test pod") + tPod.Create(ctx) + defer tPod.Cleanup(ctx) + + ginkgo.By("Checking that the test pod is running") + tPod.WaitForRunning(ctx) + + ginkgo.By("Checking that the test pod command exits with no error") + if readOnly { + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mount | grep %v | grep ro,", mountPath)) + } else { + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mount | grep %v | grep rw,", mountPath)) + } + + ginkgo.By("Checking that the gcsfuse integration tests exits with no error") + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("git clone --branch %v https://github.com/GoogleCloudPlatform/gcsfuse.git", gcsfuseTestBranch)) + + baseTestCommand := fmt.Sprintf("export GOTOOLCHAIN=go1.23.0 && export PATH=$PATH:/usr/local/go/bin && cd %v/read_cache && GODEBUG=asyncpreemptoff=1 go test . -p 1 --integrationTest -v --mountedDirectory=%v --testbucket=%v -run %v", gcsfuseIntegrationTestsBasePath, mountPath, bucketName, testName) + tPod.VerifyExecInPodSucceedWithFullOutput(f, specs.TesterContainerName, baseTestCommand) + } + + // The following test cases are derived from https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/tools/integration_tests/run_tests_mounted_directory.sh + + ginkgo.It("should succeed in TestCacheFileForRangeReadFalseTest 1", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestCacheFileForRangeReadFalseTest/TestRangeReadsWithCacheMiss", false, "50Mi", "false", "3600") + }) + + ginkgo.It("should succeed in TestCacheFileForRangeReadFalseTest 2", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestCacheFileForRangeReadFalseTest/TestConcurrentReads_ReadIsTreatedNonSequentialAfterFileIsRemovedFromCache", false, "50Mi", "false", "3600") + }) + + ginkgo.It("should succeed in TestCacheFileForRangeReadTrueTest 1", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestCacheFileForRangeReadTrueTest/TestRangeReadsWithCacheHit", false, "50Mi", "true", "3600") + }) + + ginkgo.It("should succeed in TestDisabledCacheTTLTest 1", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestDisabledCacheTTLTest/TestReadAfterObjectUpdateIsCacheMiss", false, "9Mi", "false", "0") + }) + + ginkgo.It("should succeed in TestLocalModificationTest 1", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestLocalModificationTest/TestReadAfterLocalGCSFuseWriteIsCacheMiss", false, "9Mi", "false", "3600") + }) + + ginkgo.It("should succeed in TestRangeReadTest 1", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestRangeReadTest/TestRangeReadsBeyondReadChunkSizeWithChunkDownloaded", false, "500Mi", "false", "3600") + }) + + ginkgo.It("should succeed in TestRangeReadTest 2", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestRangeReadTest/TestRangeReadsBeyondReadChunkSizeWithChunkDownloaded", false, "500Mi", "true", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 1", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestSecondSequentialReadIsCacheHit", true, "9Mi", "false", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 2", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestReadFileSequentiallyLargerThanCacheCapacity", true, "9Mi", "false", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 3", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestReadFileRandomlyLargerThanCacheCapacity", true, "9Mi", "false", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 4", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestReadMultipleFilesMoreThanCacheLimit", true, "9Mi", "false", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 5", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestReadMultipleFilesWithinCacheLimit", true, "9Mi", "false", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 6", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestSecondSequentialReadIsCacheHit", true, "9Mi", "true", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 7", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestReadFileSequentiallyLargerThanCacheCapacity", true, "9Mi", "true", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 8", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestReadFileRandomlyLargerThanCacheCapacity", true, "9Mi", "true", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 9", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestReadMultipleFilesMoreThanCacheLimit", true, "9Mi", "true", "3600") + }) + + ginkgo.It("should succeed in TestReadOnlyTest 10", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestReadOnlyTest/TestReadMultipleFilesWithinCacheLimit", true, "9Mi", "true", "3600") + }) + + ginkgo.It("should succeed in TestSmallCacheTTLTest 1", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestSmallCacheTTLTest/TestReadAfterUpdateAndCacheExpiryIsCacheMiss", false, "9Mi", "false", "10") + }) + + ginkgo.It("should succeed in TestSmallCacheTTLTest 2", func() { + init() + defer cleanup() + + gcsfuseIntegrationFileCacheTest("TestSmallCacheTTLTest/TestReadForLowMetaDataCacheTTLIsCacheHit", false, "9Mi", "false", "10") + }) +} diff --git a/test/e2e/testsuites/istio.go b/test/e2e/testsuites/istio.go index dcd7b7008..b7940d7b2 100644 --- a/test/e2e/testsuites/istio.go +++ b/test/e2e/testsuites/istio.go @@ -20,14 +20,11 @@ package testsuites import ( "context" "fmt" - "os" - "strconv" "github.com/googlecloudplatform/gcs-fuse-csi-driver/test/e2e/specs" - "github.com/googlecloudplatform/gcs-fuse-csi-driver/test/e2e/utils" "github.com/onsi/ginkgo/v2" + "google.golang.org/grpc/codes" utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/klog/v2" "k8s.io/kubernetes/test/e2e/framework" e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" storageframework "k8s.io/kubernetes/test/e2e/storage/framework" @@ -60,12 +57,6 @@ func (t *gcsFuseCSIIstioTestSuite) SkipUnsupportedTests(_ storageframework.TestD } func (t *gcsFuseCSIIstioTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { - envVar := os.Getenv(utils.TestWithNativeSidecarEnvVar) - gcsFuseSupportsNativeSidecar, err := strconv.ParseBool(envVar) - if err != nil { - klog.Fatalf(`env variable "%s" could not be converted to boolean`, envVar) - } - type local struct { config *storageframework.PerTestConfig volumeResource *storageframework.VolumeResource @@ -92,42 +83,95 @@ func (t *gcsFuseCSIIstioTestSuite) DefineTests(driver storageframework.TestDrive framework.ExpectNoError(err, "while cleaning up") } - testSidecarStoreDataSupportScenario := func(setGcsFuseNativeSidecar bool, setIstioNativeSidecar bool) { + testGCSFuseWithIstio := func(holdApplicationUntilProxyStarts, registryOnly bool) { init() defer cleanup() ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) - tPod.SetIstioSidecar(setIstioNativeSidecar) + tPod.SetLabels(map[string]string{"sidecar.istio.io/inject": "true"}) + + if holdApplicationUntilProxyStarts { + tPod.SetAnnotations(map[string]string{"proxy.istio.io/config": "{ \"holdApplicationUntilProxyStarts\": true }"}) + } + + if registryOnly { + tPod.SetAnnotations(map[string]string{"traffic.sidecar.istio.io/excludeOutboundIPRanges": "169.254.169.254/32"}) + specs.DeployIstioSidecar(f.Namespace.Name) + specs.DeployIstioServiceEntry(f.Namespace.Name) + } ginkgo.By("Deploying the pod") tPod.Create(ctx) defer tPod.Cleanup(ctx) + if !holdApplicationUntilProxyStarts { + ginkgo.By("Checking that the pod has failed container error") + tPod.WaitForFailedContainerError(ctx, "Error: failed to reserve container name") + } + ginkgo.By("Checking that the pod is running") tPod.WaitForRunning(ctx) - ginkgo.By("Checking injection ordering") - tPod.VerifyInjectionOrder(setGcsFuseNativeSidecar, setIstioNativeSidecar) - ginkgo.By("Checking that the pod command exits with no error") tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mount | grep %v | grep rw,", mountPath)) tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("echo 'hello world' > %v/data && grep 'hello world' %v/data", mountPath, mountPath)) } - // We do not support the case where: - // - GCSFuse is native sidecar. - // - Istio is a regular container. - ginkgo.It("should store data with istio regular container present", func() { - if gcsFuseSupportsNativeSidecar { - ginkgo.Skip("Unsupported case: istio-proxy is regular container while GCSFuse is an native sidecar") - } else { - testSidecarStoreDataSupportScenario(gcsFuseSupportsNativeSidecar, false /* setIstioNativeSidecar */) - } + ginkgo.It("should store data with istio injected at index 0", func() { + testGCSFuseWithIstio(true, false) + }) + + ginkgo.It("[flaky] should store data with istio injected at the last index", func() { + testGCSFuseWithIstio(false, false) + }) + + ginkgo.It("should store data with istio registry only outbound traffic policy mode", func() { + testGCSFuseWithIstio(true, true) + }) + + ginkgo.It("[flaky] should fail with istio registry only outbound traffic policy mode missing Pod annotation", func() { + init() + defer cleanup() + + ginkgo.By("Configuring the pod") + tPod := specs.NewTestPod(f.ClientSet, f.Namespace) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) + tPod.SetLabels(map[string]string{"sidecar.istio.io/inject": "true"}) + tPod.SetAnnotations(map[string]string{"proxy.istio.io/config": "{ \"holdApplicationUntilProxyStarts\": true }"}) + + specs.DeployIstioSidecar(f.Namespace.Name) + specs.DeployIstioServiceEntry(f.Namespace.Name) + + ginkgo.By("Deploying the pod") + tPod.Create(ctx) + defer tPod.Cleanup(ctx) + + ginkgo.By("Checking that the pod has failed mount error") + tPod.WaitForFailedMountError(ctx, codes.Internal.String()) + tPod.WaitForFailedMountError(ctx, "mountWithStorageHandle: fs.NewServer: create file system: SetUpBucket: Error in iterating through objects: Get") }) - ginkgo.It("should store data with istio native container present", func() { - testSidecarStoreDataSupportScenario(gcsFuseSupportsNativeSidecar, true /* setIstioNativeSidecar */) + ginkgo.It("[flaky] should fail with istio registry only outbound traffic policy mode missing ServiceEntry", func() { + init() + defer cleanup() + + ginkgo.By("Configuring the pod") + tPod := specs.NewTestPod(f.ClientSet, f.Namespace) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) + tPod.SetLabels(map[string]string{"sidecar.istio.io/inject": "true"}) + tPod.SetAnnotations(map[string]string{"proxy.istio.io/config": "{ \"holdApplicationUntilProxyStarts\": true }"}) + tPod.SetAnnotations(map[string]string{"traffic.sidecar.istio.io/excludeOutboundIPRanges": "169.254.169.254/32"}) + + specs.DeployIstioSidecar(f.Namespace.Name) + + ginkgo.By("Deploying the pod") + tPod.Create(ctx) + defer tPod.Cleanup(ctx) + + ginkgo.By("Checking that the pod has failed mount error") + tPod.WaitForFailedMountError(ctx, codes.Internal.String()) + tPod.WaitForFailedMountError(ctx, "mountWithStorageHandle: fs.NewServer: create file system: SetUpBucket: Error in iterating through objects: Get") }) } diff --git a/test/e2e/testsuites/metrics.go b/test/e2e/testsuites/metrics.go index 63ceb4f67..6ca7c3cdb 100644 --- a/test/e2e/testsuites/metrics.go +++ b/test/e2e/testsuites/metrics.go @@ -38,15 +38,15 @@ import ( ) var expectedMetricNames = map[string]int{ - "fs_ops_count": 15, + "fs_ops_count": 20, "fs_ops_error_count": 2, - "fs_ops_latency": 15, + "fs_ops_latency": 20, "gcs_download_bytes_count": 1, "gcs_read_count": 1, "gcs_read_bytes_count": 1, "gcs_reader_count": 2, - "gcs_request_count": 6, - "gcs_request_latencies": 6, + "gcs_request_count": 7, + "gcs_request_latencies": 7, "file_cache_read_count": 1, "file_cache_read_bytes_count": 1, "file_cache_read_latencies": 1, @@ -120,7 +120,7 @@ func (t *gcsFuseCSIMetricsTestSuite) DefineTests(driver storageframework.TestDri // The test driver uses config.Prefix to pass the bucket names back to the test suite. bucketName := l.config.Prefix - // Create files using gsutil + // Create a new file A outside of the gcsfuse, using gsutil. fileName := uuid.NewString() specs.CreateTestFileInBucket(fileName, bucketName) @@ -137,19 +137,65 @@ func (t *gcsFuseCSIMetricsTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Checking that the pod command exits with no error") tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mount | grep %v | grep rw,", mountPath)) + + // Read file A. + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("cat %v/%v", mountPath, fileName)) + + // Read file A again. tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("cat %v/%v", mountPath, fileName)) tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("cat %v/%v", mountPath, fileName)) + + // Create a new file B using gcsfuse. tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("touch %v/testfile", mountPath)) + + // List the volume. tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("ls %v", mountPath)) + + // Write content to file B. tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("echo 'hello world!' > %v/testfile", mountPath)) + + // Read file B. tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("cat %v/testfile", mountPath)) + + // Remove file B. tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("rm %v/testfile", mountPath)) + // Delete directory A with files in it. + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mkdir %v/dir-to-delete/", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("touch %v/dir-to-delete/my-file", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("rm -r %v/dir-to-delete/", mountPath)) + + // Copy a file A from dir1 to dir2. + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mkdir %v/my-dir/", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("touch %v/my-dir/my-file", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mkdir %v/my-other-dir/", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mv %v/my-dir/my-file %v/my-other-dir/", mountPath, mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("rmdir %v/my-dir/", mountPath)) + + // Mkdir dir A. + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mkdir %v/my-new-dir/", mountPath)) + + // Rename a file A to file B. + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("touch %v/my-file", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mv %v/my-file %v/my-renamed-file", mountPath, mountPath)) + + // Rename a directory A to directory B (set –rename-dir-limit flag to avoid i/o errors) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mkdir %v/my-dir-to-rename/", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mv %v/my-dir-to-rename/ %v/my-renamed-dir/", mountPath, mountPath)) + + // Create symlink. + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mkdir %v/my-dir/", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("touch %v/my-dir/my-file", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("ln -s %v/my-dir/my-file %v/my-symlink", mountPath, mountPath)) + + // Read symlink. + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("ls -l %v/my-symlink", mountPath)) + ginkgo.By("Sleeping 20 seconds for metrics to be collected") time.Sleep(20 * time.Second) ginkgo.By("Collecting Prometheus metrics from the CSI driver node server") - csiPodIP := tPod.GetCISDriverNodePodIP(ctx) + csiPodIP := tPod.GetCSIDriverNodePodIP(ctx) tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("wget -O %v/metrics.prom http://%v:9920/metrics", mountPath, csiPodIP)) promFile := fmt.Sprintf("%v/%v/metrics.prom", l.artifactsDir, f.Namespace.Name) diff --git a/test/e2e/utils/handler.go b/test/e2e/utils/handler.go index 07040344b..4f9b69cd1 100644 --- a/test/e2e/utils/handler.go +++ b/test/e2e/utils/handler.go @@ -68,6 +68,7 @@ type TestParameters struct { GinkgoSkipGcpSaTest bool SupportsNativeSidecar bool + IstioVersion string } const TestWithNativeSidecarEnvVar = "TEST_WITH_NATIVE_SIDECAR" @@ -173,13 +174,18 @@ func Handle(testParams *TestParameters) error { klog.Fatalf(`env variable "%s" could not be set: %v`, TestWithNativeSidecarEnvVar, err) } + testSkipStr := generateTestSkip(testParams) + if !strings.Contains(testSkipStr, "istio") && (len(testFocusStr) == 0 || strings.Contains(testFocusStr, "istio")) { + installIstio(testParams.IstioVersion) + } + //nolint:gosec cmd := exec.Command("ginkgo", "run", "-v", "--procs", testParams.GinkgoProcs, "--flake-attempts", testParams.GinkgoFlakeAttempts, "--timeout", testParams.GinkgoTimeout, "--focus", testFocusStr, - "--skip", generateTestSkip(testParams), + "--skip", testSkipStr, "--junit-report", "junit-gcsfusecsi.xml", "--output-dir", artifactsDir, testParams.PkgDir+"/test/e2e/", @@ -209,7 +215,7 @@ func generateTestSkip(testParams *TestParameters) string { } if testParams.UseGKEAutopilot { - skipTests = append(skipTests, "OOM", "high.resource.usage", "gcsfuseIntegration") + skipTests = append(skipTests, "OOM", "high.resource.usage", "gcsfuseIntegration", "istio") } if !testParams.SupportsNativeSidecar { @@ -225,9 +231,22 @@ func generateTestSkip(testParams *TestParameters) string { } } + skipTests = append(skipTests, "flaky") + skipString := strings.Join(skipTests, "|") klog.Infof("Generated ginkgo skip string: %q", skipString) return skipString } + +func installIstio(istioVersion string) { + if err := os.Setenv("ISTIO_VERSION", istioVersion); err != nil { + klog.Fatalf(`env variable "ISTIO_VERSION" could not be set: %v`, err) + } + + cmd := exec.Command("bash", "./test/e2e/utils/install-istio.sh") + if err := runCommand("Installing Istio...", cmd); err != nil { + klog.Fatalf(`failed to install Istio: %v`, err) + } +} diff --git a/test/e2e/utils/install-istio.sh b/test/e2e/utils/install-istio.sh new file mode 100755 index 000000000..85f4802a7 --- /dev/null +++ b/test/e2e/utils/install-istio.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script will install kustomize, which is a tool that simplifies patching +# Kubernetes manifests for different environments. +# https://github.com/kubernetes-sigs/kustomize + +set -o nounset +set -o errexit + +readonly INSTALL_DIR="$( dirname -- "$( readlink -f -- "$0"; )"; )/../../../bin" + +if [ ! -f "${INSTALL_DIR}" ]; then + mkdir -p "${INSTALL_DIR}" +fi + +cd ${INSTALL_DIR} + +curl -L https://istio.io/downloadIstio | sh - + +cd istio-${ISTIO_VERSION} +export PATH=$PWD/bin:$PATH + +istioctl install \ +--set profile="minimal" \ +--set values.pilot.tolerations[0].operator=Exists \ +--skip-confirmation