From 87691276e99bc5ee75086421a450ffe342553dbb Mon Sep 17 00:00:00 2001 From: Manohar Reddy Date: Tue, 13 Feb 2024 12:27:18 +0100 Subject: [PATCH] update helm charts --- .github/workflows/codeql-analysis.yaml | 66 +++++++++++++ .github/workflows/codespell.yml | 16 ++++ .github/workflows/go.yml | 36 ------- .github/workflows/linux.yaml | 33 +++++++ .github/workflows/pluto.yaml | 25 +++++ Makefile | 3 + charts/spdk-csi/templates/config-map.yaml | 4 +- .../spdk-csi/templates/controller-rbac.yaml | 8 +- charts/spdk-csi/templates/controller.yaml | 16 +++- charts/spdk-csi/templates/node.yaml | 20 +++- charts/spdk-csi/templates/storageclass.yaml | 12 ++- charts/spdk-csi/values.yaml | 46 +++++---- pkg/spdk/controllerserver.go | 4 +- test/sanity/README.md | 7 ++ test/sanity/params.yaml | 1 + test/sanity/run-test.sh | 93 +++++++++++++++++++ test/sanity/secrets.yaml | 6 ++ 17 files changed, 325 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/codeql-analysis.yaml create mode 100644 .github/workflows/codespell.yml delete mode 100644 .github/workflows/go.yml create mode 100644 .github/workflows/linux.yaml create mode 100644 .github/workflows/pluto.yaml create mode 100644 test/sanity/README.md create mode 100644 test/sanity/params.yaml create mode 100644 test/sanity/run-test.sh create mode 100644 test/sanity/secrets.yaml diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml new file mode 100644 index 0000000..c6672a7 --- /dev/null +++ b/.github/workflows/codeql-analysis.yaml @@ -0,0 +1,66 @@ + +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master, 'release-**' ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master, 'release-**' ] + schedule: + - cron: '0 */24 * * *' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: ^1.18 + id: go + + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + run: | + make all + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 0000000..9d063a6 --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,16 @@ +# GitHub Action to automate the identification of common misspellings in text files. +# https://github.com/codespell-project/actions-codespell +# https://github.com/codespell-project/codespell +name: codespell +on: [push, pull_request] +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: codespell-project/actions-codespell@master + with: + check_filenames: true + skip: ./.git,./.github/workflows/codespell.yml,.git,*.png,*.jpg,*.svg,*.sum,./vendor,go.sum + ignore_words_list: "browseable,ro" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index de40924..0000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,36 +0,0 @@ -# This workflow will build a golang project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go - -name: Go - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - -jobs: - - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: '1.20' - - - - name: Build - run: mkdir OUT_DIR - - - name: Build - run: go build -buildvcs=false -o OUT_DIR/spdkcsi ./cmd/ - - publish: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - steps: - - name: Push image - run: export OUT_DIR=OUT_DIR ; make image diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml new file mode 100644 index 0000000..7706316 --- /dev/null +++ b/.github/workflows/linux.yaml @@ -0,0 +1,33 @@ +name: Linux Unit tests +on: + pull_request: {} + push: {} + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: ^1.21 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + + - name: Test + run: | + make test + + # - name: Sanity test + # env: + # GITHUB_ACTIONS: true + # run: | + # export PATH=$PATH:$HOME/.local/bin + # make + # echo "is running in github actions: $GITHUB_ACTIONS" + # make sanity-test diff --git a/.github/workflows/pluto.yaml b/.github/workflows/pluto.yaml new file mode 100644 index 0000000..f9e03b0 --- /dev/null +++ b/.github/workflows/pluto.yaml @@ -0,0 +1,25 @@ +name: k8s api version check +on: + pull_request: {} + push: {} + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v4 + + # https://pluto.docs.fairwinds.com/advanced/#display-options + - name: Download pluto + uses: FairwindsOps/pluto/github-action@master + + - name: Check deploy folder + run: | + pluto detect-files -d deploy --ignore-deprecations --ignore-removals + - name: Check example folder + run: | + pluto detect-files -d deploy/example diff --git a/Makefile b/Makefile index 273cb61..4030fb1 100644 --- a/Makefile +++ b/Makefile @@ -145,3 +145,6 @@ image: spdkcsi clean: rm -f $(OUT_DIR)/spdkcsi go clean -testcache + +sanity-test: spdkcsi + test/sanity/run-test.sh diff --git a/charts/spdk-csi/templates/config-map.yaml b/charts/spdk-csi/templates/config-map.yaml index f88dc08..b1cbf3a 100644 --- a/charts/spdk-csi/templates/config-map.yaml +++ b/charts/spdk-csi/templates/config-map.yaml @@ -9,8 +9,8 @@ kind: ConfigMap metadata: name: spdkcsi-cm data: - # rpcURL: spdk json rpc target - # targetType: nvme-rdma, nvme-tcp, iscsi + # uuid: the simplyblock cluster UUID + # ip: the management IP of the simplyblock cluster # targetAddr: target service IP config.json: |- {{ toJson .Values.csiConfig | indent 4 -}} diff --git a/charts/spdk-csi/templates/controller-rbac.yaml b/charts/spdk-csi/templates/controller-rbac.yaml index 57c77fe..15170c2 100644 --- a/charts/spdk-csi/templates/controller-rbac.yaml +++ b/charts/spdk-csi/templates/controller-rbac.yaml @@ -39,7 +39,13 @@ rules: verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] + verbs: ["create", "get", "list", "watch", "update", "delete"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] diff --git a/charts/spdk-csi/templates/controller.yaml b/charts/spdk-csi/templates/controller.yaml index bbfccca..381c6b9 100644 --- a/charts/spdk-csi/templates/controller.yaml +++ b/charts/spdk-csi/templates/controller.yaml @@ -19,7 +19,7 @@ spec: {{ include "spdk.labels" . | indent 6 }} app: spdkcsi-controller spec: - serviceAccount: spdkcsi-controller-sa + serviceAccountName: spdkcsi-controller-sa hostNetwork: true containers: - name: spdkcsi-provisioner @@ -31,6 +31,20 @@ spec: - "--timeout=30s" - "--retry-interval-start=500ms" - "--leader-election=false" + - "--feature-gates=Topology=true" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: spdkcsi-snapshotter + image: "{{ .Values.image.csiSnapshotter.repository }}:{{ .Values.image.csiSnapshotter.tag }}" + args: + - "--csi-address=unix:///csi/csi-provisioner.sock" + - "--v=5" + - "--timeout=150s" + - "--leader-election=false" + imagePullPolicy: {{ .Values.image.csiProvisioner.pullPolicy }} + securityContext: + privileged: true volumeMounts: - name: socket-dir mountPath: /csi diff --git a/charts/spdk-csi/templates/node.yaml b/charts/spdk-csi/templates/node.yaml index 8cfe955..a8b2f61 100644 --- a/charts/spdk-csi/templates/node.yaml +++ b/charts/spdk-csi/templates/node.yaml @@ -17,7 +17,7 @@ spec: {{ include "spdk.labels" . | indent 6 }} app: spdkcsi-node spec: - serviceAccount: spdkcsi-node-sa + serviceAccountName: spdkcsi-node-sa hostNetwork: true containers: - name: spdkcsi-registrar @@ -38,7 +38,7 @@ spec: securityContext: privileged: true capabilities: - add: ["SYS_ADMIN"] + add: ["SYS_ADMIN", "SYS_MODULE"] allowPrivilegeEscalation: true image: "{{ .Values.image.spdkcsi.repository }}:{{ .Values.image.spdkcsi.tag }}" imagePullPolicy: {{ .Values.image.spdkcsi.pullPolicy }} @@ -55,8 +55,7 @@ spec: lifecycle: postStart: exec: - command: ["/bin/sh", "-c", - "/usr/sbin/iscsid || echo failed to start iscsid"] + command: ["/bin/sh", "-c", "sudo modprobe nvme-tcp || echo failed to modprobe nvme-tcp"] volumeMounts: - name: socket-dir mountPath: /csi @@ -73,6 +72,12 @@ spec: - name: spdkcsi-nodeserver-config mountPath: /etc/spdkcsi-nodeserver-config/ readOnly: true + - name: spdkcsi-config + mountPath: /etc/spdkcsi-config/ + readOnly: true + - name: spdkcsi-secret + mountPath: /etc/spdkcsi-secret/ + readOnly: true volumes: - name: socket-dir hostPath: @@ -99,3 +104,10 @@ spec: - name: spdkcsi-nodeserver-config configMap: name: spdkcsi-nodeservercm + optional: true + - name: spdkcsi-config + configMap: + name: spdkcsi-cm + - name: spdkcsi-secret + secret: + secretName: spdkcsi-secret diff --git a/charts/spdk-csi/templates/storageclass.yaml b/charts/spdk-csi/templates/storageclass.yaml index b0ffc6c..fd2477c 100644 --- a/charts/spdk-csi/templates/storageclass.yaml +++ b/charts/spdk-csi/templates/storageclass.yaml @@ -10,7 +10,17 @@ metadata: name: spdkcsi-sc provisioner: csi.spdk.io parameters: - fsType: ext4 + csi.storage.k8s.io/fstype: ext4 + pool_name: testing1 + qos_rw_iops: "0" + qos_rw_mbytes: "0" + qos_r_mbytes: "0" + qos_w_mbytes: "0" + compression: "False" + encryption: "False" + distr-ndcs: "1" + distr-npcs: "1" reclaimPolicy: Delete volumeBindingMode: Immediate +allowVolumeExpansion: true {{- end -}} diff --git a/charts/spdk-csi/values.yaml b/charts/spdk-csi/values.yaml index 7094b95..255d94f 100644 --- a/charts/spdk-csi/values.yaml +++ b/charts/spdk-csi/values.yaml @@ -4,21 +4,25 @@ --- image: spdkcsi: - repository: spdkcsi/spdkcsi - tag: canary - pullPolicy: Never + repository: manoharbrm/spdkcsi + tag: latest + pullPolicy: Always csiProvisioner: - repository: k8s.gcr.io/sig-storage/csi-provisioner - tag: v2.0.2 - pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/csi-provisioner + tag: v3.5.0 + pullPolicy: Always csiAttacher: - repository: k8s.gcr.io/sig-storage/csi-attacher - tag: v3.0.0 - pullPolicy: IfNotPresent + repository: gcr.io/k8s-staging-sig-storage/csi-attacher + tag: v3.5.0 + pullPolicy: Always nodeDriverRegistrar: - repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar - tag: v2.0.1 - pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/csi-node-driver-registrar + tag: v2.8.0 + pullPolicy: Always + csiSnapshotter: + repository: registry.k8s.io/sig-storage/csi-snapshotter + tag: v6.2.2 + pullPolicy: Always serviceAccount: # Specifies whether a serviceAccount should be created @@ -38,10 +42,9 @@ externallyManagedSecret: spdkdev: # Specifies whether a spdkdev should be created - create: true + create: false storageclass: - # Specifies whether a spdkdev should be created create: true controller: @@ -49,19 +52,14 @@ controller: # Configuration for the CSI to connect to the cluster csiConfig: - nodes: - - name: &name localhost - rpcURL: 'http://127.0.0.1:9009' - targetType: nvme-tcp - targetAddr: 127.0.0.1 + simplybk: + uuid: 963c9d0a-4506-43c3-a722-0b7c8b157038 + ip: 13.58.2.30 # Configuration for the csiSecret csiSecret: - rpcTokens: - # The "name" matches "csiConfig.node.name" - - name: *name - username: spdkcsiuser - password: spdkcsipass + simplybk: + secret: 2BAbQTPEDi4o73VHymg2 spdkdevCreateCommonds: /root/spdk/app/spdk_tgt/spdk_tgt > /tmp/spdk-tgt.log 2>&1 & diff --git a/pkg/spdk/controllerserver.go b/pkg/spdk/controllerserver.go index a70ddc1..3d27e1c 100644 --- a/pkg/spdk/controllerserver.go +++ b/pkg/spdk/controllerserver.go @@ -381,7 +381,7 @@ func NewsimplyBlockClient() (*util.NodeNVMf, error) { if err != nil { return nil, err } - klog.Infof("spdk node created: name=%s, url=%s", config.Simplybk.IP) + klog.Infof("spdk node created: url=%s", config.Simplybk.IP) return util.NewNVMf(config.Simplybk.UUID, config.Simplybk.IP, secret.Simplybk.Secret), nil } @@ -394,7 +394,7 @@ func newControllerServer(d *csicommon.CSIDriver) (*controllerServer, error) { spdkNode, err := NewsimplyBlockClient() if err != nil { - klog.Errorf("failed to create spdk node %s: %s", err.Error()) + klog.Errorf("failed to create spdk node %v", err.Error()) return nil, fmt.Errorf("no valid spdk node found") } diff --git a/test/sanity/README.md b/test/sanity/README.md new file mode 100644 index 0000000..77bcc7e --- /dev/null +++ b/test/sanity/README.md @@ -0,0 +1,7 @@ +## Sanity Tests +Testing the SMB CSI driver using the [`sanity`](https://github.com/kubernetes-csi/csi-test/tree/master/pkg/sanity) package test suite. + +### Run sanity tests +``` +make sanity-test +``` diff --git a/test/sanity/params.yaml b/test/sanity/params.yaml new file mode 100644 index 0000000..5560141 --- /dev/null +++ b/test/sanity/params.yaml @@ -0,0 +1 @@ +source: //127.0.0.1/share \ No newline at end of file diff --git a/test/sanity/run-test.sh b/test/sanity/run-test.sh new file mode 100644 index 0000000..c74ce1a --- /dev/null +++ b/test/sanity/run-test.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function cleanup { + echo 'pkill -f smbplugin' + if [ -z "$GITHUB_ACTIONS" ] + then + # if not running on github actions, do not use sudo + pkill -f smbplugin + else + # if running on github actions, use sudo + sudo pkill -f smbplugin + fi + echo 'Deleting CSI sanity test binary' + rm -rf csi-test + echo 'Uninstalling samba server on localhost' + docker rm samba -f +} + +trap cleanup EXIT + +function install_csi_sanity_bin { + echo 'Installing CSI sanity test binary...' + mkdir -p $GOPATH/src/github.com/kubernetes-csi + pushd $GOPATH/src/github.com/kubernetes-csi + export GO111MODULE=off + git clone https://github.com/kubernetes-csi/csi-test.git -b v5.0.0 + pushd csi-test/cmd/csi-sanity + make install + popd + popd +} + +function provision_simplyblock_cluster { + echo 'Running samba server on localhost' + docker run -e PERMISSIONS=0777 -p 445:445 --name samba -d andyzhangx/samba:win-fix -s "share;/smbshare/;yes;no;no;all;none" -u "sanity;sanitytestpassword" -p +} + +provision_simplyblock_cluster + +if [[ -z "$(command -v csi-sanity)" ]]; then + install_csi_sanity_bin +fi + +readonly endpoint='unix:///tmp/csi.sock' +nodeid='CSINode' +if [[ "$#" -gt 0 ]] && [[ -n "$1" ]]; then + nodeid="$1" +fi + +ARCH=$(uname -p) +if [[ "${ARCH}" == "x86_64" || ${ARCH} == "unknown" ]]; then + ARCH="amd64" +fi + +if [ -z "$GITHUB_ACTIONS" ] +then + # if not running on github actions, do not use sudo + _output/${ARCH}/smbplugin --endpoint "$endpoint" --nodeid "$nodeid" -v=5 & +else + # if running on github actions, use sudo + sudo _output/${ARCH}/smbplugin --endpoint "$endpoint" --nodeid "$nodeid" -v=5 & +fi + +# sleep a while waiting for azurefileplugin start up +sleep 1 + +echo 'Begin to run sanity test...' +CSI_SANITY_BIN=$GOPATH/bin/csi-sanity +skipTests='create a volume with already existing name and different capacity|should fail when requesting to create a volume with already existing name and different capacity|should fail when the requested volume does not exist' +if [ -z "$GITHUB_ACTIONS" ] +then + # if not running on github actions, do not use sudo + "$CSI_SANITY_BIN" --ginkgo.v --csi.secrets="$(pwd)/test/sanity/secrets.yaml" --csi.testvolumeparameters="$(pwd)/test/sanity/params.yaml" --csi.endpoint="$endpoint" --ginkgo.skip="$skipTests" +else + # if running on github actions, use sudo + sudo "$CSI_SANITY_BIN" --ginkgo.v --csi.secrets="$(pwd)/test/sanity/secrets.yaml" --csi.testvolumeparameters="$(pwd)/test/sanity/params.yaml" --csi.endpoint="$endpoint" --ginkgo.skip="$skipTests" +fi diff --git a/test/sanity/secrets.yaml b/test/sanity/secrets.yaml new file mode 100644 index 0000000..eb1e244 --- /dev/null +++ b/test/sanity/secrets.yaml @@ -0,0 +1,6 @@ +NodeStageVolumeSecret: + username: sanity + password: sanitytestpassword +CreateVolumeSecret: + username: sanity + password: sanitytestpassword