Skip to content

Commit

Permalink
Enhancements to run Trident as an AKS extension.
Browse files Browse the repository at this point in the history
*  Enhancement to run Trident as an AKS extension as part of offering Trident as an 1P service in Azure.

---------

Co-authored-by: Shubham Phadnis <42840905+sphadnis007@users.noreply.github.com>
  • Loading branch information
VinayKumarHavanur and sphadnis007 authored Apr 8, 2024
1 parent f1215ce commit fedcb29
Show file tree
Hide file tree
Showing 85 changed files with 6,492 additions and 109 deletions.
10 changes: 5 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ OPERATOR_CONFIG_PKG = github.com/netapp/trident/operator/config
TRIDENT_KUBERNETES_PKG = github.com/netapp/trident/persistent_store/crd
OPERATOR_CONFIG_PKG = github.com/netapp/trident/operator/config
OPERATOR_INSTALLER_CONFIG_PKG = github.com/netapp/trident/operator/controllers/orchestrator/installer
OPERATOR_KUBERNETES_PKG = github.com/netapp/trident/operator/controllers/orchestrator
OPERATOR_KUBERNETES_PKG = github.com/netapp/trident/operator/crd
VERSION_FILE = github.com/netapp/trident/hack/VERSION
BUILD_ROOT = /go/src/github.com/netapp/trident
TRIDENT_VOLUME = trident-build
Expand Down Expand Up @@ -429,10 +429,10 @@ k8s_codegen_operator:
@$(K8S_CODE_GENERATOR)/generate-groups.sh all $(OPERATOR_KUBERNETES_PKG)/client \
$(OPERATOR_KUBERNETES_PKG)/apis "netapp:v1" -h ./hack/boilerplate.go.txt
@rm -rf $(K8S_CODE_GENERATOR)
@rm -rf ./operator/controllers/orchestrator/client/*
@mv $(OPERATOR_KUBERNETES_PKG)/client/* ./operator/controllers/orchestrator/client/
@rm -rf ./operator/controllers/orchestrator/apis/netapp/v1/zz_generated.deepcopy.go
@mv $(OPERATOR_KUBERNETES_PKG)/apis/netapp/v1/zz_generated.deepcopy.go ./operator/controllers/orchestrator/apis/netapp/v1/
@rm -rf ./operator/crd/client/*
@mv $(OPERATOR_KUBERNETES_PKG)/client/* ./operator/crd/client/
@rm -rf ./operator/crd/apis/netapp/v1/zz_generated.deepcopy.go
@mv $(OPERATOR_KUBERNETES_PKG)/apis/netapp/v1/zz_generated.deepcopy.go ./operator/crd/apis/netapp/v1/

mocks:
@go install github.com/golang/mock/mockgen@v1.6.0
Expand Down
22 changes: 22 additions & 0 deletions cli/api/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,25 @@ type KubernetesNamespace struct {
Kind string `json:"kind"`
Metadata Metadata `json:"metadata"`
}

type CRStatus struct {
Status string `json:"status"`
Message string `json:"message"`
}

type OperatorPhaseStatus string

const (
OperatorPhaseDone OperatorPhaseStatus = "Done"
OperatorPhaseProcessing OperatorPhaseStatus = "Processing"
OperatorPhaseUnknown OperatorPhaseStatus = "Unknown"
OperatorPhaseFailed OperatorPhaseStatus = "Failed"
OperatorPhaseError OperatorPhaseStatus = "Error"
)

type OperatorStatus struct {
ErrorMessage string `json:"errorMessage"`
Status string `json:"operatorStatus"`
TorcStatus map[string]CRStatus `json:"torcStatus"`
TconfStatus map[string]CRStatus `json:"tconfStatus"`
}
20 changes: 20 additions & 0 deletions cli/cmd/check.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright 2024 NetApp, Inc. All Rights Reserved.

package cmd

import "github.com/spf13/cobra"

func init() {
RootCmd.AddCommand(checkCmd)
}

var checkCmd = &cobra.Command{
Use: "check",
Short: "check status of a Trident pod",
Hidden: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
initCmdLogging()
err := discoverOperatingMode(cmd)
return err
},
}
158 changes: 158 additions & 0 deletions cli/cmd/check_operator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
// Copyright 2024 NetApp, Inc. All Rights Reserved.

package cmd

import (
"encoding/json"
"fmt"
"net/http"
"os"
"time"

"github.com/cenkalti/backoff/v4"
"github.com/olekukonko/tablewriter"
"github.com/spf13/cobra"

cliapi "github.com/netapp/trident/cli/api"
"github.com/netapp/trident/utils/errors"
)

var checkTimeout int32

const (
RequestTimeout = 5 * time.Second
backoffMaxInterval = 5 * time.Second
operatorService = "trident-operator"
operatorServicePort = "8000"
operatorServicePath = "/operator/status"
)

func init() {
checkCmd.AddCommand(checkOperatorCmd)
checkOperatorCmd.Flags().Int32VarP(&checkTimeout, "timeout", "t", 1, "timeout in seconds for the operator check")
if err := checkOperatorCmd.Flags().MarkHidden("timeout"); err != nil {
fmt.Fprintln(os.Stderr, err)
}
}

var checkOperatorCmd = &cobra.Command{
Use: "operator",
Short: "check operator pod status",
Aliases: []string{"o"},
Hidden: true,
RunE: func(cmd *cobra.Command, args []string) error {
if OperatingMode == ModeTunnel {
command := []string{
"check",
"operator",
"--timeout",
fmt.Sprintf("%d", checkTimeout),
}
out, err := TunnelCommand(append(command, args...))
printOutput(cmd, out, err)
return err
} else {
return checkOperatorStatus(checkTimeout)
}
},
}

func checkOperatorStatus(checkTimeout int32) error {
var err error
status := cliapi.OperatorStatus{}
var response *http.Response
var url string

getOperatorStatus := func() error {
if namespace := os.Getenv("POD_NAMESPACE"); namespace == "" {
return errors.New("error in getting trident operator pod")
} else {
url = "http://" + operatorService + "." + namespace + "." + "svc.cluster.local" + ":" + operatorServicePort + operatorServicePath
}
request, err := http.NewRequest("GET", url, nil)
request.Header.Set("Content-Type", "application/json")

client := &http.Client{Timeout: RequestTimeout}
if response, err = client.Do(request); err != nil {
return err
}

defer response.Body.Close()

err = json.NewDecoder(response.Body).Decode(&status)
if status.Status != string(cliapi.OperatorPhaseDone) {
return errors.New("operator pod status is not set to done")
}

return nil
}

checkOperatorNotify := func(err error, duration time.Duration) {}

checkOperatorBackoff := backoff.NewExponentialBackOff()
checkOperatorBackoff.MaxInterval = backoffMaxInterval
checkOperatorBackoff.MaxElapsedTime = time.Duration(checkTimeout) * time.Second

if err = backoff.RetryNotify(getOperatorStatus, checkOperatorBackoff, checkOperatorNotify); err != nil {
if status.Status != "" {
writeStatus(status)
}
return err
}

writeStatus(status)
return nil
}

func writeStatus(status cliapi.OperatorStatus) {
switch OutputFormat {
case FormatJSON:
WriteJSON(status)
case FormatYAML:
WriteYAML(status)
case FormatWide:
writeStatusTableWide(status)
default:
writeStatusTable(status)
}
}

func writeStatusTable(status cliapi.OperatorStatus) {
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Name", "Type", "Status"})
if status.ErrorMessage != "" {
table.SetFooter([]string{"Overall Status", status.Status, status.ErrorMessage})
} else {
table.SetFooter([]string{"Overall Status", status.Status, ""})
}

for torcName, torcStatus := range status.TorcStatus {
table.Append([]string{torcName, "TridentOrchestratorCR", torcStatus.Status})
}

for tconfName, tconfStatus := range status.TconfStatus {
table.Append([]string{tconfName, "TridentConfiguratorCR", tconfStatus.Status})
}

table.Render()
}

func writeStatusTableWide(status cliapi.OperatorStatus) {
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Name", "Type", "Status", "Message"})
if status.ErrorMessage != "" {
table.SetFooter([]string{"Overall Status", status.Status, status.ErrorMessage, ""})
} else {
table.SetFooter([]string{"Overall Status", status.Status, "", ""})
}

for torcName, torcStatus := range status.TorcStatus {
table.Append([]string{torcName, "TridentOrchestratorCR", torcStatus.Status, torcStatus.Message})
}

for tconfName, tconfStatus := range status.TconfStatus {
table.Append([]string{tconfName, "TridentConfiguratorCR", tconfStatus.Status, tconfStatus.Message})
}

table.Render()
}
2 changes: 2 additions & 0 deletions cli/cmd/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ const (
VolumeCRDName = "tridentvolumes.trident.netapp.io"
VolumePublicationCRDName = "tridentvolumepublications.trident.netapp.io"
VolumeReferenceCRDName = "tridentvolumereferences.trident.netapp.io"
ConfiguratorCRDName = "tridentconfigurators.trident.netapp.io"

ControllerRoleFilename = "trident-controller-role.yaml"
ControllerClusterRoleFilename = "trident-controller-clusterrole.yaml"
Expand Down Expand Up @@ -181,6 +182,7 @@ var (
VolumeCRDName,
VolumePublicationCRDName,
ActionSnapshotRestoreCRDName,
ConfiguratorCRDName,
}
)

Expand Down
2 changes: 1 addition & 1 deletion cli/k8s_client/client_factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (

"github.com/netapp/trident/config"
. "github.com/netapp/trident/logging"
torc "github.com/netapp/trident/operator/controllers/orchestrator/client/clientset/versioned"
torc "github.com/netapp/trident/operator/crd/client/clientset/versioned"
tridentv1clientset "github.com/netapp/trident/persistent_store/crd/client/clientset/versioned"
"github.com/netapp/trident/utils/errors"
)
Expand Down
69 changes: 68 additions & 1 deletion cli/k8s_client/yaml_factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,10 @@ spec:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://plugin/csi.sock
- name: TRIDENT_SERVER
Expand Down Expand Up @@ -1597,6 +1601,12 @@ func GetBackendConfigCRDYAML() string {
return tridentBackendConfigCRDYAMLv1
}

func GetConfiguratorCRDYAML() string {
Log().Trace(">>>> GetConfiguratorCRDYAML")
defer func() { Log().Trace("<<<< GetConfiguratorCRDYAML") }()
return tridentConfiguratorCRDYAMLv1
}

func GetMirrorRelationshipCRDYAML() string {
Log().Trace(">>>> GetMirrorRelationshipCRDYAML")
defer func() { Log().Trace("<<<< GetMirrorRelationshipCRDYAML") }()
Expand Down Expand Up @@ -2066,6 +2076,62 @@ spec:
- trident-internal
- trident-external`

const tridentConfiguratorCRDYAMLv1 = `
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: tridentconfigurators.trident.netapp.io
spec:
group: trident.netapp.io
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
subresources:
status: {}
additionalPrinterColumns:
- name: Phase
type: string
description: The backend config phase
priority: 0
jsonPath: .status.phase
- name: Status
type: string
description: The result of the last operation
priority: 0
jsonPath: .status.lastOperationStatus
- name: Cloud Provider
type: string
description: The name of cloud provider
priority: 0
jsonPath: .status.cloudProvider
- name: Storage Driver
type: string
description: The storage driver type
priority: 1
jsonPath: .spec.storageDriverName
- name: Deletion Policy
type: string
description: The deletion policy
priority: 1
jsonPath: .status.deletionPolicy
scope: Cluster
names:
plural: tridentconfigurators
singular: tridentconfigurator
kind: TridentConfigurator
shortNames:
- tconf
- tconfigurator
categories:
- trident
- trident-internal
- trident-external`

const tridentStorageClassCRDYAMLv1 = `
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
Expand Down Expand Up @@ -2433,7 +2499,8 @@ const customResourceDefinitionYAMLv1 = tridentVersionCRDYAMLv1 +
"\n---" + tridentTransactionCRDYAMLv1 +
"\n---" + tridentSnapshotCRDYAMLv1 +
"\n---" + tridentVolumeReferenceCRDYAMLv1 +
"\n---" + tridentActionSnapshotRestoreCRDYAMLv1 + "\n"
"\n---" + tridentActionSnapshotRestoreCRDYAMLv1 +
"\n---" + tridentConfiguratorCRDYAMLv1 + "\n"

func GetCSIDriverYAML(name string, labels, controllingCRDetails map[string]string) string {
Log().WithFields(LogFields{
Expand Down
Loading

0 comments on commit fedcb29

Please sign in to comment.