From 548819dfd564c10c1b09018d8d43bff96fc06304 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Thu, 7 Sep 2023 16:39:58 +0530 Subject: [PATCH 01/64] Modified to supportVault HA --- charts/vault-cred/templates/deployment.yaml | 3 +++ charts/vault-cred/values.yaml | 1 + config/config.go | 1 + internal/client/vault.go | 9 +++++++++ internal/job/vault_seal_watcher.go | 15 +++++++++++++++ 5 files changed, 29 insertions(+) diff --git a/charts/vault-cred/templates/deployment.yaml b/charts/vault-cred/templates/deployment.yaml index 1525eda1..6ffac606 100644 --- a/charts/vault-cred/templates/deployment.yaml +++ b/charts/vault-cred/templates/deployment.yaml @@ -1,3 +1,4 @@ + apiVersion: apps/v1 kind: Deployment metadata: @@ -42,6 +43,8 @@ spec: value: "{{ .Values.env.logLevel }}" - name: VAULT_ADDR value: "{{ .Values.vault.vaultAddress }}" + - name: HA_ENABLED + value: "{{ .Values.vault.haenabled }}" - name: VAULT_SECRET_NAME value: "{{ .Values.vault.secretName }}" - name: VAULT_SECRET_TOKEN_KEY_NAME diff --git a/charts/vault-cred/values.yaml b/charts/vault-cred/values.yaml index 4cdae5f4..861b59c5 100644 --- a/charts/vault-cred/values.yaml +++ b/charts/vault-cred/values.yaml @@ -44,6 +44,7 @@ env: logLevel: info vault: + haenabled: true vaultAddress: http://vault:8200 secretName: vault-server secretTokenKeyName: roottoken diff --git a/config/config.go b/config/config.go index db1ed67b..d3733237 100644 --- a/config/config.go +++ b/config/config.go @@ -15,6 +15,7 @@ type Configuration struct { } type VaultEnv struct { + HAEnabled bool `envconfig:"HA_ENABLED"` Address string `envconfig:"VAULT_ADDR" required:"true"` CACert string `envconfig:"VAULT_CACERT" required:"false"` ReadTimeout time.Duration `envconfig:"VAULT_READ_TIMEOUT" default:"60s"` diff --git a/internal/client/vault.go b/internal/client/vault.go index 1edcd62a..3ca2f8d2 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -175,3 +175,12 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa } return } + +func (vc *VaultClient) JoinRaftCluster() error { + + req := api.RaftJoinRequest{ + LeaderAPIAddr: "https://vault-0.vault-internal:8200", + } + _, err := vc.c.Sys().RaftJoin(&req) // Replace with your leader address + return err +} diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 35d8bcc3..1c3b2fc1 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -41,9 +41,24 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to get vault seal status, %s", err) return } + if (v.conf.HAEnabled){ + v.log.Infof("HA ENABLED",v.conf.HAEnabled) + err:=vc.JoinRaftCluster() + if (err!=nil){ + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + + } + } if res { v.log.Info("vault is sealed, trying to unseal") + if (v.conf.HAEnabled){ + err := vc.JoinRaftCluster() + if (err!=nil){ + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + + } + } err := vc.Unseal() if err != nil { v.log.Errorf("failed to unseal vault, %s", err) From 086eac32a91b44dcd5c884ae619456fef98bf174 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Thu, 7 Sep 2023 17:08:58 +0530 Subject: [PATCH 02/64] Leader Address Changed --- internal/client/vault.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index 3ca2f8d2..4e5f7acc 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -177,9 +177,9 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa } func (vc *VaultClient) JoinRaftCluster() error { - + req := api.RaftJoinRequest{ - LeaderAPIAddr: "https://vault-0.vault-internal:8200", + LeaderAPIAddr: "https://capten-dev-vault-0.capten-dev-vault-internal:8201", } _, err := vc.c.Sys().RaftJoin(&req) // Replace with your leader address return err From 92a59d3c163db2dc42db2baef49e350cab5d0eed Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Thu, 7 Sep 2023 21:22:33 +0530 Subject: [PATCH 03/64] Added Raft --- config/config.go | 2 +- internal/job/vault_seal_watcher.go | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/config/config.go b/config/config.go index d3733237..e524e2db 100644 --- a/config/config.go +++ b/config/config.go @@ -15,7 +15,7 @@ type Configuration struct { } type VaultEnv struct { - HAEnabled bool `envconfig:"HA_ENABLED"` + HAEnabled bool `envconfig:"HA_ENABLED" default:"true"` Address string `envconfig:"VAULT_ADDR" required:"true"` CACert string `envconfig:"VAULT_CACERT" required:"false"` ReadTimeout time.Duration `envconfig:"VAULT_READ_TIMEOUT" default:"60s"` diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 1c3b2fc1..df1e98d5 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -41,23 +41,24 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to get vault seal status, %s", err) return } - if (v.conf.HAEnabled){ - v.log.Infof("HA ENABLED",v.conf.HAEnabled) - err:=vc.JoinRaftCluster() - if (err!=nil){ + if v.conf.HAEnabled { + v.log.Infof("HA ENABLED", v.conf.HAEnabled) + err := vc.JoinRaftCluster() + if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) - - } + + } } if res { v.log.Info("vault is sealed, trying to unseal") - if (v.conf.HAEnabled){ + if v.conf.HAEnabled { + v.log.Infof("HA Enabled", v.conf.HAEnabled) err := vc.JoinRaftCluster() - if (err!=nil){ + if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) - - } + + } } err := vc.Unseal() if err != nil { From d93c4f730a8176d1212fae4a5d63e02b87b868bd Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 8 Sep 2023 13:28:57 +0530 Subject: [PATCH 04/64] Updated Raft --- internal/client/vault.go | 2 ++ internal/job/vault_seal_watcher.go | 43 ++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index 4e5f7acc..0b05cfed 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -179,6 +179,8 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa func (vc *VaultClient) JoinRaftCluster() error { req := api.RaftJoinRequest{ + Retry: true, + LeaderAPIAddr: "https://capten-dev-vault-0.capten-dev-vault-internal:8201", } _, err := vc.c.Sys().RaftJoin(&req) // Replace with your leader address diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index df1e98d5..de84b757 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -41,25 +41,25 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to get vault seal status, %s", err) return } - if v.conf.HAEnabled { - v.log.Infof("HA ENABLED", v.conf.HAEnabled) - err := vc.JoinRaftCluster() - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) + // if v.conf.HAEnabled { + // v.log.Infof("HA ENABLED", v.conf.HAEnabled) + // err := vc.JoinRaftCluster() + // if err != nil { + // v.log.Errorf("Failed to join the HA cluster: %v\n", err) - } - } + // } + // } if res { v.log.Info("vault is sealed, trying to unseal") - if v.conf.HAEnabled { - v.log.Infof("HA Enabled", v.conf.HAEnabled) - err := vc.JoinRaftCluster() - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) + // if v.conf.HAEnabled { + // v.log.Infof("HA Enabled", v.conf.HAEnabled) + // err := vc.JoinRaftCluster() + // if err != nil { + // v.log.Errorf("Failed to join the HA cluster: %v\n", err) - } - } + // } + // } err := vc.Unseal() if err != nil { v.log.Errorf("failed to unseal vault, %s", err) @@ -68,6 +68,21 @@ func (v *VaultSealWatcher) Run() { v.log.Info("vault unsealed executed") res, err := vc.IsVaultSealed() + if res { + if v.conf.HAEnabled { + v.log.Infof("HA Enabled", v.conf.HAEnabled) + err := vc.JoinRaftCluster() + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + + } + } + err := vc.Unseal() + if err != nil { + v.log.Errorf("failed to unseal vault, %s", err) + return + } + } if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) return From 83901c12b153d15d0fcc0d7a72fde9fd9477a514 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 8 Sep 2023 23:55:13 +0530 Subject: [PATCH 05/64] Updated Raft --- internal/job/vault_seal_watcher.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index de84b757..b20482a9 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -41,14 +41,14 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to get vault seal status, %s", err) return } - // if v.conf.HAEnabled { - // v.log.Infof("HA ENABLED", v.conf.HAEnabled) - // err := vc.JoinRaftCluster() - // if err != nil { - // v.log.Errorf("Failed to join the HA cluster: %v\n", err) + if v.conf.HAEnabled { + v.log.Infof("HA ENABLED", v.conf.HAEnabled) + err := vc.JoinRaftCluster() + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) - // } - // } + } + } if res { v.log.Info("vault is sealed, trying to unseal") From 82ef5724b168124fcddc8bcc5375ca5fe3faf704 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sat, 9 Sep 2023 00:46:00 +0530 Subject: [PATCH 06/64] Updated Raft --- internal/client/vault.go | 13 ++-- internal/client/vault_seal.go | 74 +++++++++++++++++++ internal/job/vault_seal_watcher.go | 111 ++++++++++++++++++----------- 3 files changed, 152 insertions(+), 46 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index 0b05cfed..432ac314 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -177,12 +177,17 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa } func (vc *VaultClient) JoinRaftCluster() error { - + leaderInfo, err := vc.c.Sys().Leader() + if err != nil { + vc.log.Debug("Failed to retrieve leader information: %v\n", err) + + } req := api.RaftJoinRequest{ Retry: true, - - LeaderAPIAddr: "https://capten-dev-vault-0.capten-dev-vault-internal:8201", + + LeaderAPIAddr: leaderInfo.LeaderAddress, } - _, err := vc.c.Sys().RaftJoin(&req) // Replace with your leader address + vc.log.Debug("Leader API address: %s\n", leaderInfo.LeaderAddress) + _, err = vc.c.Sys().RaftJoin(&req) // Replace with your leader address return err } diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 234d293e..1e95c44e 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -27,6 +27,7 @@ func (vc *VaultClient) Unseal() error { return nil } + rootToken, unsealKeys, err := vc.getVaultSecretValues() if err != nil { return err @@ -121,3 +122,76 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { } return rootToken, unsealKeys, nil } + +func (vc *VaultClient)UnsealVaultInstance(svc string, unsealKey string) error { + // Create a Vault API client + address :=fmt.Sprintf("http://%s:8200", svc) + err:=vc.c.SetAddress(address) + if (err!=nil){ + vc.log.Errorf("Error while setting address") + } + vc.log.Debug("Address",address) + + // Check if Vault is sealed and unseal if necessary + + // Vault is sealed; unseal it + unsealResponse, err :=vc.c.Sys().Unseal(unsealKey) + if err != nil { + return err + } + + if unsealResponse.Sealed { + vc.log.Debug("Vault is still sealed after unsealing attempt") + } + + // You can add additional error handling or log responses as needed + return nil +} + + + + + +func (vc *VaultClient) GetVaultSecretValuesforMultiInstance() (string, []string, error) { + k8s, err := NewK8SClient(vc.log) + if err != nil { + return "", nil, errors.WithMessage(err, "error initializing k8s client") + } + + vaultSec, err := k8s.GetSecret(context.Background(), vc.conf.VaultSecretName, vc.conf.VaultSecretNameSpace) + if err != nil { + if strings.Contains(err.Error(), "secret not found") { + vc.log.Debugf("secret %d not found", vc.conf.VaultSecretName) + return "", nil, nil + } + return "", nil, errors.WithMessage(err, "error fetching vault secret") + } + + vc.log.Debugf("found %d vault secret values", len(vaultSec.Data)) + unsealKeys := []string{} + var rootToken string + for key, val := range vaultSec.Data { + if strings.HasPrefix(key, vc.conf.VaultSecretUnSealKeyPrefix) { + unsealKeys = append(unsealKeys, val) + continue + } + if strings.EqualFold(key, vc.conf.VaultSecretTokenKeyName) { + rootToken = val + } + } + return rootToken, unsealKeys, nil +} + + +func (vc *VaultClient) IsVaultSealedForAllInstances(svc string) (bool, error) { + address :=fmt.Sprintf("http://%s:8200", svc) + err:=vc.c.SetAddress(address) + if (err!=nil){ + vc.log.Errorf("Error while setting address") + } + status, err := vc.c.Sys().SealStatus() + if err != nil { + return false, err + } + return status.Sealed, nil +} \ No newline at end of file diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index b20482a9..a86f22f1 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -35,61 +35,88 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("%s", err) return } + servicename := []string{"capten-dev-vault-0", "capten-dev-vault-1", "capten-dev-vault-2"} + // res, err := vc.IsVaultSealed() + // if err != nil { + // v.log.Errorf("failed to get vault seal status, %s", err) + // return + // } - res, err := vc.IsVaultSealed() - if err != nil { - v.log.Errorf("failed to get vault seal status, %s", err) - return - } if v.conf.HAEnabled { v.log.Infof("HA ENABLED", v.conf.HAEnabled) - err := vc.JoinRaftCluster() - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - - } - } - - if res { - v.log.Info("vault is sealed, trying to unseal") - // if v.conf.HAEnabled { - // v.log.Infof("HA Enabled", v.conf.HAEnabled) - // err := vc.JoinRaftCluster() - // if err != nil { - // v.log.Errorf("Failed to join the HA cluster: %v\n", err) - - // } + // res, err := vc.IsVaultSealed() + // if err != nil { + // v.log.Errorf("failed to get vault seal status, %s", err) + // return // } - err := vc.Unseal() - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return - } - v.log.Info("vault unsealed executed") + for _, svc := range servicename { + // Extract the pod's IP address + res, err := vc.IsVaultSealedForAllInstances(svc) + if err != nil { + v.log.Errorf("failed to get vault seal status, %s", err) + return + } + if res { + if svc == "capten-dev-vault-0" { + vc.Unseal() + } else { + err := vc.JoinRaftCluster() + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + + } + _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + if err != nil { + v.log.Errorf("Failed to fetch the credential: %v\n", err) - res, err := vc.IsVaultSealed() - if res { - if v.conf.HAEnabled { - v.log.Infof("HA Enabled", v.conf.HAEnabled) - err := vc.JoinRaftCluster() - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) + } + key := unsealKeys[0] + vc.UnsealVaultInstance(svc, key) } } - err := vc.Unseal() - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return - } + + //res, err = vc.IsVaultSealed() + // Perform the unseal operation on the Vault instance within the pod using the podIP } + + } + for _, svc := range servicename { + res, err := vc.IsVaultSealedForAllInstances(svc) + + v.log.Debug("Seal Status of %v :%v", svc, res) if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) return } v.log.Infof("vault sealed status: %v", res) - return - } else { - v.log.Debug("vault is in unsealed status") } + + // if res { + // v.log.Info("vault is sealed, trying to unseal") + + // err := vc.Unseal() + // if err != nil { + // v.log.Errorf("failed to unseal vault, %s", err) + // return + // } + // v.log.Info("vault unsealed executed") + + // res, err := vc.IsVaultSealed() + // if res { + // err := vc.Unseal() + // if err != nil { + // v.log.Errorf("failed to unseal vault, %s", err) + // return + // } + // } + // if err != nil { + // v.log.Errorf("failed to get vault seal status, %s", err) + // return + // } + // v.log.Infof("vault sealed status: %v", res) + // return + // } else { + // v.log.Debug("vault is in unsealed status") + // } } From b8a66fccb3c6edc6a5c69973d8ee0f1241e54898 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sat, 9 Sep 2023 00:54:29 +0530 Subject: [PATCH 07/64] Updated Raft --- internal/client/vault.go | 4 ++-- internal/job/vault_seal_watcher.go | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index 432ac314..ba4c0b58 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -182,12 +182,12 @@ func (vc *VaultClient) JoinRaftCluster() error { vc.log.Debug("Failed to retrieve leader information: %v\n", err) } - req := api.RaftJoinRequest{ + req := &api.RaftJoinRequest{ Retry: true, LeaderAPIAddr: leaderInfo.LeaderAddress, } vc.log.Debug("Leader API address: %s\n", leaderInfo.LeaderAddress) - _, err = vc.c.Sys().RaftJoin(&req) // Replace with your leader address + _, err = vc.c.Sys().RaftJoin(req) // Replace with your leader address return err } diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index a86f22f1..d20ac3a7 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -63,12 +63,13 @@ func (v *VaultSealWatcher) Run() { err := vc.JoinRaftCluster() if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return } _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) - + return } key := unsealKeys[0] vc.UnsealVaultInstance(svc, key) @@ -91,7 +92,7 @@ func (v *VaultSealWatcher) Run() { } v.log.Infof("vault sealed status: %v", res) } - + // if res { // v.log.Info("vault is sealed, trying to unseal") @@ -115,7 +116,7 @@ func (v *VaultSealWatcher) Run() { // return // } // v.log.Infof("vault sealed status: %v", res) - // return + // } else { // v.log.Debug("vault is in unsealed status") // } From 2b6f0d898eb90a7827514375c33563aa7bb410d1 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sat, 9 Sep 2023 00:59:56 +0530 Subject: [PATCH 08/64] Updated Raft --- internal/client/vault.go | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index ba4c0b58..6e80fae4 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -177,17 +177,38 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa } func (vc *VaultClient) JoinRaftCluster() error { + var req *api.RaftJoinRequest leaderInfo, err := vc.c.Sys().Leader() if err != nil { - vc.log.Debug("Failed to retrieve leader information: %v\n", err) + vc.log.Debugf("Failed to retrieve leader information: %v", err) + return err } - req := &api.RaftJoinRequest{ - Retry: true, - LeaderAPIAddr: leaderInfo.LeaderAddress, + if leaderInfo.LeaderAddress == "" { + // Handle the case where leader address is empty + vc.log.Debug("Leader address is empty") + return err + } else { + req = &api.RaftJoinRequest{ + Retry: true, + LeaderAPIAddr: leaderInfo.LeaderAddress, + } } - vc.log.Debug("Leader API address: %s\n", leaderInfo.LeaderAddress) + + // req := &api.RaftJoinRequest{ + // Retry: true, + // LeaderAPIAddr: leaderInfo.LeaderAddress, + // } + + vc.log.Debugf("Leader API address: %s", leaderInfo.LeaderAddress) + _, err = vc.c.Sys().RaftJoin(req) // Replace with your leader address - return err + if err != nil { + // Handle the join error (e.g., return it or log and return) + vc.log.Debugf("Failed to join the Raft cluster: %v", err) + return err + } + + return nil } From 5c6dca9ca818eb9898886344abbed78c699beb27 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sat, 9 Sep 2023 01:12:41 +0530 Subject: [PATCH 09/64] Updated Raft --- internal/job/vault_seal_watcher.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index d20ac3a7..7ff5fe0b 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -60,12 +60,6 @@ func (v *VaultSealWatcher) Run() { if svc == "capten-dev-vault-0" { vc.Unseal() } else { - err := vc.JoinRaftCluster() - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - return - - } _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) @@ -75,6 +69,12 @@ func (v *VaultSealWatcher) Run() { vc.UnsealVaultInstance(svc, key) } + err := vc.JoinRaftCluster() + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return + + } } //res, err = vc.IsVaultSealed() From 5448ae6a4d175346481bda9fa1903ddd71c9720f Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sun, 10 Sep 2023 15:08:21 +0530 Subject: [PATCH 10/64] Updated Raft --- internal/client/vault_seal.go | 44 ++++++++++++++---------------- internal/job/vault_seal_watcher.go | 2 +- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 1e95c44e..284d599f 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -27,7 +27,6 @@ func (vc *VaultClient) Unseal() error { return nil } - rootToken, unsealKeys, err := vc.getVaultSecretValues() if err != nil { return err @@ -123,35 +122,32 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { return rootToken, unsealKeys, nil } -func (vc *VaultClient)UnsealVaultInstance(svc string, unsealKey string) error { +func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey string) error { // Create a Vault API client - address :=fmt.Sprintf("http://%s:8200", svc) - err:=vc.c.SetAddress(address) - if (err!=nil){ + vc.log.Debug("Checking Unseal status for vault Instance") + address := fmt.Sprintf("http://%s:8200", svc) + err := vc.c.SetAddress(address) + if err != nil { vc.log.Errorf("Error while setting address") } - vc.log.Debug("Address",address) + vc.log.Debug("Address", address) // Check if Vault is sealed and unseal if necessary - // Vault is sealed; unseal it - unsealResponse, err :=vc.c.Sys().Unseal(unsealKey) - if err != nil { - return err - } + // Vault is sealed; unseal it + unsealResponse, err := vc.c.Sys().Unseal(unsealKey) + if err != nil { + return err + } + + if unsealResponse.Sealed { + vc.log.Debug("Vault is still sealed after unsealing attempt") + } - if unsealResponse.Sealed { - vc.log.Debug("Vault is still sealed after unsealing attempt") - } - // You can add additional error handling or log responses as needed return nil } - - - - func (vc *VaultClient) GetVaultSecretValuesforMultiInstance() (string, []string, error) { k8s, err := NewK8SClient(vc.log) if err != nil { @@ -182,11 +178,11 @@ func (vc *VaultClient) GetVaultSecretValuesforMultiInstance() (string, []string, return rootToken, unsealKeys, nil } - func (vc *VaultClient) IsVaultSealedForAllInstances(svc string) (bool, error) { - address :=fmt.Sprintf("http://%s:8200", svc) - err:=vc.c.SetAddress(address) - if (err!=nil){ + address := fmt.Sprintf("http://%s:8200", svc) + err := vc.c.SetAddress(address) + vc.log.Debug("Address for checking vault status", address) + if err != nil { vc.log.Errorf("Error while setting address") } status, err := vc.c.Sys().SealStatus() @@ -194,4 +190,4 @@ func (vc *VaultClient) IsVaultSealedForAllInstances(svc string) (bool, error) { return false, err } return status.Sealed, nil -} \ No newline at end of file +} diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 7ff5fe0b..e893bc18 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -50,7 +50,7 @@ func (v *VaultSealWatcher) Run() { // return // } for _, svc := range servicename { - // Extract the pod's IP address + res, err := vc.IsVaultSealedForAllInstances(svc) if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) From a589046b37d362249272614f5dc97857954b9dbd Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sun, 10 Sep 2023 15:15:29 +0530 Subject: [PATCH 11/64] Updated Raft --- internal/job/vault_seal_watcher.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index e893bc18..5c269bed 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -50,7 +50,7 @@ func (v *VaultSealWatcher) Run() { // return // } for _, svc := range servicename { - + res, err := vc.IsVaultSealedForAllInstances(svc) if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) @@ -63,18 +63,19 @@ func (v *VaultSealWatcher) Run() { _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) - return + return } key := unsealKeys[0] vc.UnsealVaultInstance(svc, key) + err = vc.JoinRaftCluster() + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return - } - err := vc.JoinRaftCluster() - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - return + } } + } //res, err = vc.IsVaultSealed() @@ -92,7 +93,7 @@ func (v *VaultSealWatcher) Run() { } v.log.Infof("vault sealed status: %v", res) } - + // if res { // v.log.Info("vault is sealed, trying to unseal") @@ -116,7 +117,7 @@ func (v *VaultSealWatcher) Run() { // return // } // v.log.Infof("vault sealed status: %v", res) - + // } else { // v.log.Debug("vault is in unsealed status") // } From 0ecfd01b2e63eafdf1378ab9d8dc17354b2a3bfe Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sun, 10 Sep 2023 15:27:05 +0530 Subject: [PATCH 12/64] Updated Raft --- internal/client/vault_seal.go | 28 +++++++++++++++++++--------- internal/job/vault_seal_watcher.go | 3 +-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 284d599f..c61b3b14 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -122,7 +122,7 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { return rootToken, unsealKeys, nil } -func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey string) error { +func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey []string) error { // Create a Vault API client vc.log.Debug("Checking Unseal status for vault Instance") address := fmt.Sprintf("http://%s:8200", svc) @@ -132,17 +132,27 @@ func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey string) error { } vc.log.Debug("Address", address) + for _, key := range unsealKey { + unsealResponse, err := vc.c.Sys().Unseal(key) + if err != nil { + return errors.WithMessage(err, "error while unsealing") + } + if unsealResponse.Sealed { + vc.log.Debug("Vault is still sealed after unsealing attempt") + } + } + // Check if Vault is sealed and unseal if necessary // Vault is sealed; unseal it - unsealResponse, err := vc.c.Sys().Unseal(unsealKey) - if err != nil { - return err - } - - if unsealResponse.Sealed { - vc.log.Debug("Vault is still sealed after unsealing attempt") - } + // unsealResponse, err := vc.c.Sys().Unseal(unsealKey) + // if err != nil { + // return err + // } + + // if unsealResponse.Sealed { + // vc.log.Debug("Vault is still sealed after unsealing attempt") + // } // You can add additional error handling or log responses as needed return nil diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 5c269bed..5411fc71 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -65,8 +65,7 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - key := unsealKeys[0] - vc.UnsealVaultInstance(svc, key) + vc.UnsealVaultInstance(svc, unsealKeys) err = vc.JoinRaftCluster() if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) From 8eeba216e4fadb81b3a1072347ae13a72400e166 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sun, 10 Sep 2023 22:36:14 +0530 Subject: [PATCH 13/64] Updated Raft --- config/config.go | 2 ++ internal/client/vault_seal.go | 2 +- internal/job/vault_seal_watcher.go | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index e524e2db..558ec0af 100644 --- a/config/config.go +++ b/config/config.go @@ -17,6 +17,8 @@ type Configuration struct { type VaultEnv struct { HAEnabled bool `envconfig:"HA_ENABLED" default:"true"` Address string `envconfig:"VAULT_ADDR" required:"true"` + Address2 string `envconfig:"VAULT_ADDR2" default:"vault-cred-sync-data"` + Adddress3 string `envconfig:"VAULT_ADDR3" default:"vault-cred-sync-data"` CACert string `envconfig:"VAULT_CACERT" required:"false"` ReadTimeout time.Duration `envconfig:"VAULT_READ_TIMEOUT" default:"60s"` MaxRetries int `envconfig:"VAULT_MAX_RETRIES" default:"5"` diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index c61b3b14..d50d186a 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -125,7 +125,7 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey []string) error { // Create a Vault API client vc.log.Debug("Checking Unseal status for vault Instance") - address := fmt.Sprintf("http://%s:8200", svc) + address := fmt.Sprintf("https://%s:8200", svc) err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 5411fc71..3a150b69 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -30,6 +30,7 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") + vc, err := client.NewVaultClient(v.log, v.conf) if err != nil { v.log.Errorf("%s", err) From 54c0e844da8d42829f29635fb959de6a3b45bc03 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sun, 10 Sep 2023 22:40:33 +0530 Subject: [PATCH 14/64] Updated Raft --- internal/client/vault_seal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index d50d186a..2f36c384 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -125,7 +125,7 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey []string) error { // Create a Vault API client vc.log.Debug("Checking Unseal status for vault Instance") - address := fmt.Sprintf("https://%s:8200", svc) + address := fmt.Sprintf("https://%s.capten-dev-vault-internal:8200", svc) err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") From 0a855e92f3ed1b38d40666f0db2224a37af95565 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sun, 10 Sep 2023 23:22:32 +0530 Subject: [PATCH 15/64] Updated Raft --- config/config.go | 6 +- internal/client/vault_seal.go | 3 +- internal/job/vault_seal_watcher.go | 171 ++++++++++++++++++----------- 3 files changed, 109 insertions(+), 71 deletions(-) diff --git a/config/config.go b/config/config.go index 558ec0af..7728a387 100644 --- a/config/config.go +++ b/config/config.go @@ -16,9 +16,9 @@ type Configuration struct { type VaultEnv struct { HAEnabled bool `envconfig:"HA_ENABLED" default:"true"` - Address string `envconfig:"VAULT_ADDR" required:"true"` - Address2 string `envconfig:"VAULT_ADDR2" default:"vault-cred-sync-data"` - Adddress3 string `envconfig:"VAULT_ADDR3" default:"vault-cred-sync-data"` + Address string `envconfig:"VAULT_ADDR" default:"http://capten-dev-vault-0:8200"` + Address2 string `envconfig:"VAULT_ADDR2" default:"http://capten-dev-vault-1:8200"` + Adddress3 string `envconfig:"VAULT_ADDR3" default:"http://capten-dev-vault-2:8200"` CACert string `envconfig:"VAULT_CACERT" required:"false"` ReadTimeout time.Duration `envconfig:"VAULT_READ_TIMEOUT" default:"60s"` MaxRetries int `envconfig:"VAULT_MAX_RETRIES" default:"5"` diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 2f36c384..395e2213 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -125,7 +125,7 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey []string) error { // Create a Vault API client vc.log.Debug("Checking Unseal status for vault Instance") - address := fmt.Sprintf("https://%s.capten-dev-vault-internal:8200", svc) + address := fmt.Sprintf("https://%s:8200", svc) err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") @@ -154,7 +154,6 @@ func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey []string) error // vc.log.Debug("Vault is still sealed after unsealing attempt") // } - // You can add additional error handling or log responses as needed return nil } diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 3a150b69..daa8543e 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -31,94 +31,133 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") - vc, err := client.NewVaultClient(v.log, v.conf) - if err != nil { - v.log.Errorf("%s", err) - return + addresses := []string{ + v.conf.Address, + v.conf.Address2, + v.conf.Adddress3, } + + //vc, err := client.NewVaultClient(v.log, v.conf) + // if err != nil { + // v.log.Errorf("%s", err) + // return + // } servicename := []string{"capten-dev-vault-0", "capten-dev-vault-1", "capten-dev-vault-2"} - // res, err := vc.IsVaultSealed() + //res, err := vc.IsVaultSealed() // if err != nil { // v.log.Errorf("failed to get vault seal status, %s", err) // return // } + var vc *client.VaultClient + var vaultClients []*client.VaultClient + for _, address := range addresses { + conf := config.VaultEnv{ + Address: address, + ReadTimeout: 30, + MaxRetries: 3, + // Set other configuration options as needed + } + v.log.Debug("Address Configuration", conf) + + vc, err := client.NewVaultClient(v.log, v.conf) + + if err != nil { + v.log.Errorf("%s", err) + return + } + + vaultClients = append(vaultClients, vc) + } if v.conf.HAEnabled { + v.log.Infof("HA ENABLED", v.conf.HAEnabled) - // res, err := vc.IsVaultSealed() - // if err != nil { - // v.log.Errorf("failed to get vault seal status, %s", err) - // return - // } + for _, svc := range servicename { + switch svc { + case "capten-dev-vault-0": + vc = vaultClients[0] // Use the first Vault client + case "capten-dev-vault-1": + vc = vaultClients[1] // Use the second Vault client + case "capten-dev-vault-2": + vc = vaultClients[2] // Use the third Vault client + default: + // Handle the case where the service name doesn't match any of the instances + } - res, err := vc.IsVaultSealedForAllInstances(svc) + // res, err := vc.IsVaultSealedForAllInstances(svc) + // if err != nil { + // v.log.Errorf("failed to get vault seal status, %s", err) + // return + // } + // if res { + // if svc == "capten-dev-vault-0" { + // vc.Unseal() + // } else { + // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + // if err != nil { + // v.log.Errorf("Failed to fetch the credential: %v\n", err) + // return + // } + // vc.UnsealVaultInstance(svc, unsealKeys) + // err = vc.JoinRaftCluster() + // if err != nil { + // v.log.Errorf("Failed to join the HA cluster: %v\n", err) + // return + + // } + + // } + + // } + + // //res, err = vc.IsVaultSealed() + // // Perform the unseal operation on the Vault instance within the pod using the podIP + // } + + // } + // for _, svc := range servicename { + // res, err := vc.IsVaultSealedForAllInstances(svc) + + // v.log.Debug("Seal Status of %v :%v", svc, res) + // if err != nil { + // v.log.Errorf("failed to get vault seal status, %s", err) + // return + // } + // v.log.Infof("vault sealed status: %v", res) + // } + res, err := vc.IsVaultSealed() if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) return } if res { - if svc == "capten-dev-vault-0" { - vc.Unseal() - } else { - _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - if err != nil { - v.log.Errorf("Failed to fetch the credential: %v\n", err) - return - } - vc.UnsealVaultInstance(svc, unsealKeys) - err = vc.JoinRaftCluster() + v.log.Info("vault is sealed, trying to unseal") + + err := vc.Unseal() + if err != nil { + v.log.Errorf("failed to unseal vault, %s", err) + return + } + v.log.Info("vault unsealed executed") + + res, err := vc.IsVaultSealed() + if res { + err := vc.Unseal() if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) + v.log.Errorf("failed to unseal vault, %s", err) return - } - } + if err != nil { + v.log.Errorf("failed to get vault seal status, %s", err) + return + } + v.log.Infof("vault sealed status: %v", res) + } else { + v.log.Debug("vault is in unsealed status") } - - //res, err = vc.IsVaultSealed() - // Perform the unseal operation on the Vault instance within the pod using the podIP } - - } - for _, svc := range servicename { - res, err := vc.IsVaultSealedForAllInstances(svc) - - v.log.Debug("Seal Status of %v :%v", svc, res) - if err != nil { - v.log.Errorf("failed to get vault seal status, %s", err) - return - } - v.log.Infof("vault sealed status: %v", res) } - - // if res { - // v.log.Info("vault is sealed, trying to unseal") - - // err := vc.Unseal() - // if err != nil { - // v.log.Errorf("failed to unseal vault, %s", err) - // return - // } - // v.log.Info("vault unsealed executed") - - // res, err := vc.IsVaultSealed() - // if res { - // err := vc.Unseal() - // if err != nil { - // v.log.Errorf("failed to unseal vault, %s", err) - // return - // } - // } - // if err != nil { - // v.log.Errorf("failed to get vault seal status, %s", err) - // return - // } - // v.log.Infof("vault sealed status: %v", res) - - // } else { - // v.log.Debug("vault is in unsealed status") - // } } From 53d9e29f82ccb1a06f7f83a10345f95bad170b97 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 09:14:38 +0530 Subject: [PATCH 16/64] Updated Raft --- internal/client/vault.go | 1 + internal/job/vault_seal_watcher.go | 97 +++++++++++------------------- 2 files changed, 35 insertions(+), 63 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index 6e80fae4..a6b996f8 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -72,6 +72,7 @@ func NewVaultClientForVaultToken(log logging.Logger, conf config.VaultEnv) (*Vau func NewVaultClient(log logging.Logger, conf config.VaultEnv) (*VaultClient, error) { cfg, err := prepareVaultConfig(conf) if err != nil { + log.Debug("Error while preparing vault Config") return nil, err } diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index daa8543e..25b044f2 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -36,18 +36,8 @@ func (v *VaultSealWatcher) Run() { v.conf.Address2, v.conf.Adddress3, } - - //vc, err := client.NewVaultClient(v.log, v.conf) - // if err != nil { - // v.log.Errorf("%s", err) - // return - // } servicename := []string{"capten-dev-vault-0", "capten-dev-vault-1", "capten-dev-vault-2"} - //res, err := vc.IsVaultSealed() - // if err != nil { - // v.log.Errorf("failed to get vault seal status, %s", err) - // return - // } + var vc *client.VaultClient var vaultClients []*client.VaultClient for _, address := range addresses { @@ -55,6 +45,7 @@ func (v *VaultSealWatcher) Run() { Address: address, ReadTimeout: 30, MaxRetries: 3, + // Set other configuration options as needed } v.log.Debug("Address Configuration", conf) @@ -68,6 +59,7 @@ func (v *VaultSealWatcher) Run() { vaultClients = append(vaultClients, vc) } + v.log.Debug("Vault Clients",vaultClients) if v.conf.HAEnabled { @@ -76,56 +68,17 @@ func (v *VaultSealWatcher) Run() { for _, svc := range servicename { switch svc { case "capten-dev-vault-0": - vc = vaultClients[0] // Use the first Vault client + vc = vaultClients[0] + v.log.Debug("Vault Client:",vc) case "capten-dev-vault-1": - vc = vaultClients[1] // Use the second Vault client + vc = vaultClients[1] + v.log.Debug("Vault Client:",vc) case "capten-dev-vault-2": - vc = vaultClients[2] // Use the third Vault client + vc = vaultClients[2] + v.log.Debug("Vault Client:",vc) default: // Handle the case where the service name doesn't match any of the instances - } - - // res, err := vc.IsVaultSealedForAllInstances(svc) - // if err != nil { - // v.log.Errorf("failed to get vault seal status, %s", err) - // return - // } - // if res { - // if svc == "capten-dev-vault-0" { - // vc.Unseal() - // } else { - // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - // if err != nil { - // v.log.Errorf("Failed to fetch the credential: %v\n", err) - // return - // } - // vc.UnsealVaultInstance(svc, unsealKeys) - // err = vc.JoinRaftCluster() - // if err != nil { - // v.log.Errorf("Failed to join the HA cluster: %v\n", err) - // return - - // } - - // } - - // } - - // //res, err = vc.IsVaultSealed() - // // Perform the unseal operation on the Vault instance within the pod using the podIP - // } - - // } - // for _, svc := range servicename { - // res, err := vc.IsVaultSealedForAllInstances(svc) - - // v.log.Debug("Seal Status of %v :%v", svc, res) - // if err != nil { - // v.log.Errorf("failed to get vault seal status, %s", err) - // return - // } - // v.log.Infof("vault sealed status: %v", res) - // } + } res, err := vc.IsVaultSealed() if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) @@ -133,22 +86,40 @@ func (v *VaultSealWatcher) Run() { } if res { v.log.Info("vault is sealed, trying to unseal") + if svc == "capten-dev-vault-0" { + vc.Unseal() + }else { + err := vc.Unseal() + if err != nil { + v.log.Errorf("failed to unseal vault, %s", err) + return + } + v.log.Info("vault unsealed executed") + err = vc.JoinRaftCluster() + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return - err := vc.Unseal() - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return + } } - v.log.Info("vault unsealed executed") + + // err := vc.Unseal() + // if err != nil { + // v.log.Errorf("failed to unseal vault, %s", err) + // return + // } + res, err := vc.IsVaultSealed() if res { + err := vc.Unseal() if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return } - } + + } if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) return From e361aa94f706ffb25cb7c5a7dc94ee29a892f4db Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 09:30:29 +0530 Subject: [PATCH 17/64] Updated Raft --- charts/vault-cred/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/vault-cred/values.yaml b/charts/vault-cred/values.yaml index 861b59c5..5e8d730a 100644 --- a/charts/vault-cred/values.yaml +++ b/charts/vault-cred/values.yaml @@ -45,7 +45,7 @@ env: vault: haenabled: true - vaultAddress: http://vault:8200 + vaultAddress: http://capten-dev-vault-0:8200 secretName: vault-server secretTokenKeyName: roottoken secretUnSealKeyPrefix: unsealkey From ab20db461a85260a56c97c8eadcd736e989ceb2d Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 09:51:50 +0530 Subject: [PATCH 18/64] Updated Raft --- internal/client/vault_seal.go | 3 +- internal/job/vault_seal_watcher.go | 76 +++++++++++++++++++----------- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 395e2213..0059ea4b 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -18,6 +18,7 @@ func (vc *VaultClient) IsVaultSealed() (bool, error) { } func (vc *VaultClient) Unseal() error { + status, err := vc.c.Sys().SealStatus() if err != nil { return err @@ -125,7 +126,7 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey []string) error { // Create a Vault API client vc.log.Debug("Checking Unseal status for vault Instance") - address := fmt.Sprintf("https://%s:8200", svc) + address := fmt.Sprintf("http://%s:8200", svc) err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 25b044f2..a6a5056b 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -59,7 +59,7 @@ func (v *VaultSealWatcher) Run() { vaultClients = append(vaultClients, vc) } - v.log.Debug("Vault Clients",vaultClients) + v.log.Debug("Vault Clients", vaultClients) if v.conf.HAEnabled { @@ -68,17 +68,17 @@ func (v *VaultSealWatcher) Run() { for _, svc := range servicename { switch svc { case "capten-dev-vault-0": - vc = vaultClients[0] - v.log.Debug("Vault Client:",vc) + vc = vaultClients[0] + v.log.Debug("Vault Client:", vc) case "capten-dev-vault-1": - vc = vaultClients[1] - v.log.Debug("Vault Client:",vc) + vc = vaultClients[1] + v.log.Debug("Vault Client:", vc) case "capten-dev-vault-2": vc = vaultClients[2] - v.log.Debug("Vault Client:",vc) + v.log.Debug("Vault Client:", vc) default: // Handle the case where the service name doesn't match any of the instances - } + } res, err := vc.IsVaultSealed() if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) @@ -87,13 +87,24 @@ func (v *VaultSealWatcher) Run() { if res { v.log.Info("vault is sealed, trying to unseal") if svc == "capten-dev-vault-0" { - vc.Unseal() - }else { - err := vc.Unseal() + _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) + v.log.Errorf("Failed to fetch the credential: %v\n", err) return } + vc.UnsealVaultInstance(svc, unsealKeys) + } else { + _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + if err != nil { + v.log.Errorf("Failed to fetch the credential: %v\n", err) + return + } + vc.UnsealVaultInstance(svc, unsealKeys) + // err := vc.Unseal() + // if err != nil { + // v.log.Errorf("failed to unseal vault, %s", err) + // return + // } v.log.Info("vault unsealed executed") err = vc.JoinRaftCluster() if err != nil { @@ -108,27 +119,38 @@ func (v *VaultSealWatcher) Run() { // v.log.Errorf("failed to unseal vault, %s", err) // return // } - - res, err := vc.IsVaultSealed() - if res { + // res, err := vc.IsVaultSealed() + // if res { - err := vc.Unseal() - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return - } + // err := vc.Unseal() + // if err != nil { + // v.log.Errorf("failed to unseal vault, %s", err) + // return + // } - } - if err != nil { - v.log.Errorf("failed to get vault seal status, %s", err) - return - } - v.log.Infof("vault sealed status: %v", res) + // } + // if err != nil { + // v.log.Errorf("failed to get vault seal status, %s", err) + // return + // } + // v.log.Infof("vault sealed status: %v", res) - } else { - v.log.Debug("vault is in unsealed status") + // } else { + // v.log.Debug("vault is in unsealed status") + // } + } + + } + for _, svc := range servicename { + res, err := vc.IsVaultSealedForAllInstances(svc) + + v.log.Debug("Seal Status of %v :%v", svc, res) + if err != nil { + v.log.Errorf("failed to get vault seal status, %s", err) + return } + v.log.Infof("vault sealed status: %v", res) } } } From ba6b88ee1b42c23467fcaab533f759543c749873 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 10:23:03 +0530 Subject: [PATCH 19/64] Updated Raft --- internal/job/vault_seal_watcher.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index a6a5056b..04c1fcb0 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -92,20 +92,24 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - vc.UnsealVaultInstance(svc, unsealKeys) + err = vc.UnsealVaultInstance(svc, unsealKeys) + if err != nil { + v.log.Errorf("failed to unseal vault, %s", err) + return + } } else { _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - vc.UnsealVaultInstance(svc, unsealKeys) + err = vc.UnsealVaultInstance(svc, unsealKeys) // err := vc.Unseal() - // if err != nil { - // v.log.Errorf("failed to unseal vault, %s", err) - // return - // } - v.log.Info("vault unsealed executed") + if err != nil { + v.log.Errorf("failed to unseal vault, %s", err) + return + } + // v.log.Info("vault unsealed executed") err = vc.JoinRaftCluster() if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) From 1de8bb136b70e251543379e31c1eab716c81bca1 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 10:49:51 +0530 Subject: [PATCH 20/64] Updated Raft --- internal/job/vault_seal_watcher.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 04c1fcb0..b42330cf 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -87,24 +87,28 @@ func (v *VaultSealWatcher) Run() { if res { v.log.Info("vault is sealed, trying to unseal") if svc == "capten-dev-vault-0" { - _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - if err != nil { - v.log.Errorf("Failed to fetch the credential: %v\n", err) - return - } - err = vc.UnsealVaultInstance(svc, unsealKeys) + // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + // v.log.Debug("Unseal Keys",unsealKeys) + // if err != nil { + // v.log.Errorf("Failed to fetch the credential: %v\n", err) + // return + // } + err := vc.Unseal() + // err = vc.UnsealVaultInstance(svc, unsealKeys) if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return } + } else { _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + v.log.Debug("Unseal Keys", unsealKeys) if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - err = vc.UnsealVaultInstance(svc, unsealKeys) - // err := vc.Unseal() + // err = vc.UnsealVaultInstance(svc, unsealKeys) + err = vc.Unseal() if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return From 4471e2572f05985910129ca81b66fa7435e953ac Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 11:23:58 +0530 Subject: [PATCH 21/64] Updated Raft --- internal/client/vault_seal.go | 16 ++++++++++++++-- internal/job/vault_seal_watcher.go | 20 ++++++++++---------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 0059ea4b..f650d87c 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -2,6 +2,7 @@ package client import ( "context" + "encoding/base64" "fmt" "strings" @@ -178,11 +179,22 @@ func (vc *VaultClient) GetVaultSecretValuesforMultiInstance() (string, []string, var rootToken string for key, val := range vaultSec.Data { if strings.HasPrefix(key, vc.conf.VaultSecretUnSealKeyPrefix) { - unsealKeys = append(unsealKeys, val) + decodedValue, err := base64.StdEncoding.DecodeString(val) + if err != nil { + return "", nil, errors.WithMessage(err, "error decoding value") + } + + unsealKeys = append(unsealKeys, string(decodedValue)) + vc.log.Debug("Unseal Keys", unsealKeys) continue } if strings.EqualFold(key, vc.conf.VaultSecretTokenKeyName) { - rootToken = val + decodedValue, err := base64.StdEncoding.DecodeString(val) + if err != nil { + return "", nil, errors.WithMessage(err, "error decoding root token") + } + rootToken = string(decodedValue) + vc.log.Debug("Root Token Key", rootToken) } } return rootToken, unsealKeys, nil diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index b42330cf..bb1130c7 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -87,14 +87,14 @@ func (v *VaultSealWatcher) Run() { if res { v.log.Info("vault is sealed, trying to unseal") if svc == "capten-dev-vault-0" { - // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - // v.log.Debug("Unseal Keys",unsealKeys) - // if err != nil { - // v.log.Errorf("Failed to fetch the credential: %v\n", err) - // return - // } - err := vc.Unseal() - // err = vc.UnsealVaultInstance(svc, unsealKeys) + _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + v.log.Debug("Unseal Keys",unsealKeys) + if err != nil { + v.log.Errorf("Failed to fetch the credential: %v\n", err) + return + } + // err := vc.Unseal() + err = vc.UnsealVaultInstance(svc, unsealKeys) if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return @@ -107,8 +107,8 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - // err = vc.UnsealVaultInstance(svc, unsealKeys) - err = vc.Unseal() + err = vc.UnsealVaultInstance(svc, unsealKeys) + //err = vc.Unseal() if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return From dd420072a463eccb82a428f92e7fe63b9552646c Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 11:58:52 +0530 Subject: [PATCH 22/64] Updated Raft --- internal/client/vault_seal.go | 10 ++-- internal/job/vault_seal_watcher.go | 78 ++++++++++++++++-------------- 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index f650d87c..4d5d7945 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -2,7 +2,7 @@ package client import ( "context" - "encoding/base64" + //"encoding/base64" "fmt" "strings" @@ -179,21 +179,21 @@ func (vc *VaultClient) GetVaultSecretValuesforMultiInstance() (string, []string, var rootToken string for key, val := range vaultSec.Data { if strings.HasPrefix(key, vc.conf.VaultSecretUnSealKeyPrefix) { - decodedValue, err := base64.StdEncoding.DecodeString(val) + // decodedValue, err := base64.StdEncoding.DecodeString(val) if err != nil { return "", nil, errors.WithMessage(err, "error decoding value") } - unsealKeys = append(unsealKeys, string(decodedValue)) + unsealKeys = append(unsealKeys,val) vc.log.Debug("Unseal Keys", unsealKeys) continue } if strings.EqualFold(key, vc.conf.VaultSecretTokenKeyName) { - decodedValue, err := base64.StdEncoding.DecodeString(val) + // decodedValue, err := base64.StdEncoding.DecodeString(val) if err != nil { return "", nil, errors.WithMessage(err, "error decoding root token") } - rootToken = string(decodedValue) + rootToken = val vc.log.Debug("Root Token Key", rootToken) } } diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index bb1130c7..42b767a1 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -30,56 +30,60 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") - - addresses := []string{ - v.conf.Address, - v.conf.Address2, - v.conf.Adddress3, + vc, err := client.NewVaultClient(v.log, v.conf) + if err != nil { + v.log.Errorf("%s", err) + return } + // addresses := []string{ + // v.conf.Address, + // v.conf.Address2, + // v.conf.Adddress3, + // } servicename := []string{"capten-dev-vault-0", "capten-dev-vault-1", "capten-dev-vault-2"} - var vc *client.VaultClient - var vaultClients []*client.VaultClient - for _, address := range addresses { - conf := config.VaultEnv{ - Address: address, - ReadTimeout: 30, - MaxRetries: 3, + // var vc *client.VaultClient + // var vaultClients []*client.VaultClient + // for _, address := range addresses { + // conf := config.VaultEnv{ + // Address: address, + // ReadTimeout: 30, + // MaxRetries: 3, - // Set other configuration options as needed - } - v.log.Debug("Address Configuration", conf) + // // Set other configuration options as needed + // } + // v.log.Debug("Address Configuration", conf) - vc, err := client.NewVaultClient(v.log, v.conf) + // vc, err := client.NewVaultClient(v.log, v.conf) - if err != nil { - v.log.Errorf("%s", err) - return - } + // if err != nil { + // v.log.Errorf("%s", err) + // return + // } - vaultClients = append(vaultClients, vc) - } - v.log.Debug("Vault Clients", vaultClients) + // vaultClients = append(vaultClients, vc) + // } + // v.log.Debug("Vault Clients", vaultClients) if v.conf.HAEnabled { v.log.Infof("HA ENABLED", v.conf.HAEnabled) for _, svc := range servicename { - switch svc { - case "capten-dev-vault-0": - vc = vaultClients[0] - v.log.Debug("Vault Client:", vc) - case "capten-dev-vault-1": - vc = vaultClients[1] - v.log.Debug("Vault Client:", vc) - case "capten-dev-vault-2": - vc = vaultClients[2] - v.log.Debug("Vault Client:", vc) - default: - // Handle the case where the service name doesn't match any of the instances - } - res, err := vc.IsVaultSealed() + // switch svc { + // case "capten-dev-vault-0": + // vc = vaultClients[0] + // v.log.Debug("Vault Client:", vc) + // case "capten-dev-vault-1": + // vc = vaultClients[1] + // v.log.Debug("Vault Client:", vc) + // case "capten-dev-vault-2": + // vc = vaultClients[2] + // v.log.Debug("Vault Client:", vc) + // default: + // // Handle the case where the service name doesn't match any of the instances + // } + res, err := vc.IsVaultSealedForAllInstances(svc) if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) return From b8c249e97771573bdce7f1350d79f7f5b4b954ef Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 12:26:42 +0530 Subject: [PATCH 23/64] Updated Raft --- internal/client/vault_seal.go | 27 ++++++++++++++++++++++----- internal/job/vault_seal_watcher.go | 7 ++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 4d5d7945..5f5c2859 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/vault/api" "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (vc *VaultClient) IsVaultSealed() (bool, error) { @@ -124,10 +125,10 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { return rootToken, unsealKeys, nil } -func (vc *VaultClient) UnsealVaultInstance(svc string, unsealKey []string) error { +func (vc *VaultClient) UnsealVaultInstance(podip string, unsealKey []string) error { // Create a Vault API client vc.log.Debug("Checking Unseal status for vault Instance") - address := fmt.Sprintf("http://%s:8200", svc) + address := fmt.Sprintf("http://%s:8200", podip) err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") @@ -171,6 +172,7 @@ func (vc *VaultClient) GetVaultSecretValuesforMultiInstance() (string, []string, vc.log.Debugf("secret %d not found", vc.conf.VaultSecretName) return "", nil, nil } + return "", nil, errors.WithMessage(err, "error fetching vault secret") } @@ -179,17 +181,17 @@ func (vc *VaultClient) GetVaultSecretValuesforMultiInstance() (string, []string, var rootToken string for key, val := range vaultSec.Data { if strings.HasPrefix(key, vc.conf.VaultSecretUnSealKeyPrefix) { - // decodedValue, err := base64.StdEncoding.DecodeString(val) + // decodedValue, err := base64.StdEncoding.DecodeString(val) if err != nil { return "", nil, errors.WithMessage(err, "error decoding value") } - unsealKeys = append(unsealKeys,val) + unsealKeys = append(unsealKeys, val) vc.log.Debug("Unseal Keys", unsealKeys) continue } if strings.EqualFold(key, vc.conf.VaultSecretTokenKeyName) { - // decodedValue, err := base64.StdEncoding.DecodeString(val) + // decodedValue, err := base64.StdEncoding.DecodeString(val) if err != nil { return "", nil, errors.WithMessage(err, "error decoding root token") } @@ -213,3 +215,18 @@ func (vc *VaultClient) IsVaultSealedForAllInstances(svc string) (bool, error) { } return status.Sealed, nil } + +func (vc *VaultClient) GetPodIP(podName, namespace string) (string, error) { + k8s, err := NewK8SClient(vc.log) + if err != nil { + return "", errors.WithMessage(err, "error initializing k8s client") + } + + // Get the pod's IP address + pod, err := k8s.client.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + return "", err + } + vc.log.Debug("Pod ip", pod.Status.PodIP) + return pod.Status.PodIP, nil +} diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 42b767a1..f78053a0 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -97,8 +97,13 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } + podip,err:=vc.GetPodIP(svc,"platform") + if err != nil { + v.log.Errorf("failed to retrieve pod ip, %s", err) + return + } // err := vc.Unseal() - err = vc.UnsealVaultInstance(svc, unsealKeys) + err = vc.UnsealVaultInstance(podip, unsealKeys) if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return From 3bf8dc6015b57c93e9aa75064b7c3438a1154474 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 12:33:09 +0530 Subject: [PATCH 24/64] Updated Raft --- internal/job/vault_seal_watcher.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index f78053a0..2de9d3ac 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -116,7 +116,12 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - err = vc.UnsealVaultInstance(svc, unsealKeys) + podip,err:=vc.GetPodIP(svc,"platform") + if err != nil { + v.log.Errorf("failed to retrieve pod ip, %s", err) + return + } + err = vc.UnsealVaultInstance(podip, unsealKeys) //err = vc.Unseal() if err != nil { v.log.Errorf("failed to unseal vault, %s", err) From 9887478eef97eb6eff48bdb56d4ab0bd683991a7 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 12:42:43 +0530 Subject: [PATCH 25/64] Updated Raft --- internal/client/vault_seal.go | 2 +- internal/job/vault_seal_watcher.go | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 5f5c2859..08ff2ae1 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -128,7 +128,7 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { func (vc *VaultClient) UnsealVaultInstance(podip string, unsealKey []string) error { // Create a Vault API client vc.log.Debug("Checking Unseal status for vault Instance") - address := fmt.Sprintf("http://%s:8200", podip) + address := fmt.Sprintf("https://%s:8200", podip) err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 2de9d3ac..30a0b847 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -110,6 +110,12 @@ func (v *VaultSealWatcher) Run() { } } else { + err = vc.JoinRaftCluster() + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return + + } _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() v.log.Debug("Unseal Keys", unsealKeys) if err != nil { @@ -128,12 +134,12 @@ func (v *VaultSealWatcher) Run() { return } // v.log.Info("vault unsealed executed") - err = vc.JoinRaftCluster() - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - return + // err = vc.JoinRaftCluster() + // if err != nil { + // v.log.Errorf("Failed to join the HA cluster: %v\n", err) + // return - } + // } } // err := vc.Unseal() From 67593f5443a46042c1ea6e1a9c5125a4bbcc3d3f Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 12:53:01 +0530 Subject: [PATCH 26/64] Updated Raft --- internal/client/vault.go | 10 +++++++++- internal/job/vault_seal_watcher.go | 5 +++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index a6b996f8..88ed719b 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -3,6 +3,7 @@ package client import ( "context" "encoding/base64" + "fmt" "github.com/hashicorp/go-retryablehttp" "github.com/hashicorp/vault/api" @@ -177,8 +178,15 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa return } -func (vc *VaultClient) JoinRaftCluster() error { +func (vc *VaultClient) JoinRaftCluster(podip string) error { var req *api.RaftJoinRequest + address := fmt.Sprintf("https://%s:8200", podip) + err := vc.c.SetAddress(address) + if err != nil { + vc.log.Errorf("Error while setting address") + } + vc.log.Debug("Address", address) + leaderInfo, err := vc.c.Sys().Leader() if err != nil { diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 30a0b847..c34d55fd 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -110,7 +110,8 @@ func (v *VaultSealWatcher) Run() { } } else { - err = vc.JoinRaftCluster() + podip,err:=vc.GetPodIP(svc,"platform") + err = vc.JoinRaftCluster(podip) if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) return @@ -122,7 +123,7 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - podip,err:=vc.GetPodIP(svc,"platform") + //podip,err:=vc.GetPodIP(svc,"platform") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return From 04a0ab78def42ecac5dea0aeb40ea1c1965f0410 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 13:10:30 +0530 Subject: [PATCH 27/64] Updated Raft --- internal/client/vault.go | 2 +- internal/client/vault_seal.go | 2 +- internal/job/vault_seal_watcher.go | 11 ++++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index 88ed719b..c9d5fef9 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -180,7 +180,7 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa func (vc *VaultClient) JoinRaftCluster(podip string) error { var req *api.RaftJoinRequest - address := fmt.Sprintf("https://%s:8200", podip) + address := fmt.Sprintf("http://%s:8200", podip) err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 08ff2ae1..5f5c2859 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -128,7 +128,7 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { func (vc *VaultClient) UnsealVaultInstance(podip string, unsealKey []string) error { // Create a Vault API client vc.log.Debug("Checking Unseal status for vault Instance") - address := fmt.Sprintf("https://%s:8200", podip) + address := fmt.Sprintf("http://%s:8200", podip) err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index c34d55fd..57549904 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -83,7 +83,12 @@ func (v *VaultSealWatcher) Run() { // default: // // Handle the case where the service name doesn't match any of the instances // } - res, err := vc.IsVaultSealedForAllInstances(svc) + podip,err:=vc.GetPodIP(svc,"platform") + if err != nil { + v.log.Errorf("failed to retrieve pod ip, %s", err) + return + } + res, err := vc.IsVaultSealedForAllInstances(podip) if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) return @@ -111,6 +116,10 @@ func (v *VaultSealWatcher) Run() { } else { podip,err:=vc.GetPodIP(svc,"platform") + if err != nil { + v.log.Errorf("failed to retrieve pod ip, %s", err) + return + } err = vc.JoinRaftCluster(podip) if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) From 278c43eaf884c493179a224074a2eed54b144ac4 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 11 Sep 2023 13:58:17 +0530 Subject: [PATCH 28/64] Updated Raft --- internal/client/vault_seal.go | 53 ++++++++++++++++++++++++++++++ internal/job/vault_seal_watcher.go | 33 ++++++++----------- 2 files changed, 67 insertions(+), 19 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 5f5c2859..c028cb87 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -230,3 +230,56 @@ func (vc *VaultClient) GetPodIP(podName, namespace string) (string, error) { vc.log.Debug("Pod ip", pod.Status.PodIP) return pod.Status.PodIP, nil } + + + + + + + + + + +func (vc *VaultClient) RetrieveKeys(nameSpace string, SecretName string) (string, []string, error) { + var values []string + var rootToken string + clientset,err := NewK8SClient(vc.log) + if err != nil { + return "", nil, errors.WithMessage(err, "error initializing k8s client") + } + namespace := nameSpace // Namespace where you want to create the Secret + secretName := SecretName + secret2, err := clientset.client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + fmt.Println("Error while getting secret", err) + } + + for key, value := range secret2.Data { + + if key == "roottoken" { + rootToken = string(value) + + continue // Skip the last element + } + + vc.log.Debug("Retrieved value for %s: %s\n", key, value) + keys := string(value) + values = append(values, keys) + // fmt.Println("Key is ", keys) + + } + fmt.Println("Values", values) + fmt.Println("RootToken", rootToken) + if (secret2.Name != "") && (secret2.Namespace != "") { + vc.log.Debug("Secret '%s' found in namespace '%s'\n", secret2.Name, secret2.Namespace) + } else { + vc.log.Debug("Given Namespace and Secret Name not found") + } + // Use the secret as needed + // for _, key := range rootToken { + // fmt.Println("Root Token", key) + // } + // Mount the unseal key and root token to the Vault path + + return rootToken,values, nil +} \ No newline at end of file diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 57549904..1b2c785b 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -83,7 +83,7 @@ func (v *VaultSealWatcher) Run() { // default: // // Handle the case where the service name doesn't match any of the instances // } - podip,err:=vc.GetPodIP(svc,"platform") + podip, err := vc.GetPodIP(svc, "platform") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return @@ -96,18 +96,18 @@ func (v *VaultSealWatcher) Run() { if res { v.log.Info("vault is sealed, trying to unseal") if svc == "capten-dev-vault-0" { - _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - v.log.Debug("Unseal Keys",unsealKeys) + _, unsealKeys, err := vc.RetrieveKeys("platform", v.conf.VaultSecretName) + v.log.Debug("Unseal Keys", unsealKeys) if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - podip,err:=vc.GetPodIP(svc,"platform") + podip, err := vc.GetPodIP(svc, "platform") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return } - // err := vc.Unseal() + // err := vc.Unseal() err = vc.UnsealVaultInstance(podip, unsealKeys) if err != nil { v.log.Errorf("failed to unseal vault, %s", err) @@ -115,18 +115,12 @@ func (v *VaultSealWatcher) Run() { } } else { - podip,err:=vc.GetPodIP(svc,"platform") + podip, err := vc.GetPodIP(svc, "platform") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return } - err = vc.JoinRaftCluster(podip) - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - return - - } - _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + _, unsealKeys, err := vc.RetrieveKeys("platform", v.conf.VaultSecretName) v.log.Debug("Unseal Keys", unsealKeys) if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) @@ -144,12 +138,12 @@ func (v *VaultSealWatcher) Run() { return } // v.log.Info("vault unsealed executed") - // err = vc.JoinRaftCluster() - // if err != nil { - // v.log.Errorf("Failed to join the HA cluster: %v\n", err) - // return + err = vc.JoinRaftCluster(podip) + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return - // } + } } // err := vc.Unseal() @@ -181,7 +175,8 @@ func (v *VaultSealWatcher) Run() { } for _, svc := range servicename { - res, err := vc.IsVaultSealedForAllInstances(svc) + podip, _ := vc.GetPodIP(svc, "platform") + res, err := vc.IsVaultSealedForAllInstances(podip) v.log.Debug("Seal Status of %v :%v", svc, res) if err != nil { From 0341b96b56eeae28a4acc38f9be68ab002c16118 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 12 Sep 2023 09:41:36 +0530 Subject: [PATCH 29/64] Updated with new Vault client --- internal/client/vault_seal.go | 43 ---------------- internal/job/vault_seal_watcher.go | 82 +++++++++++++++--------------- 2 files changed, 41 insertions(+), 84 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index c028cb87..febdca39 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -240,46 +240,3 @@ func (vc *VaultClient) GetPodIP(podName, namespace string) (string, error) { -func (vc *VaultClient) RetrieveKeys(nameSpace string, SecretName string) (string, []string, error) { - var values []string - var rootToken string - clientset,err := NewK8SClient(vc.log) - if err != nil { - return "", nil, errors.WithMessage(err, "error initializing k8s client") - } - namespace := nameSpace // Namespace where you want to create the Secret - secretName := SecretName - secret2, err := clientset.client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) - if err != nil { - fmt.Println("Error while getting secret", err) - } - - for key, value := range secret2.Data { - - if key == "roottoken" { - rootToken = string(value) - - continue // Skip the last element - } - - vc.log.Debug("Retrieved value for %s: %s\n", key, value) - keys := string(value) - values = append(values, keys) - // fmt.Println("Key is ", keys) - - } - fmt.Println("Values", values) - fmt.Println("RootToken", rootToken) - if (secret2.Name != "") && (secret2.Namespace != "") { - vc.log.Debug("Secret '%s' found in namespace '%s'\n", secret2.Name, secret2.Namespace) - } else { - vc.log.Debug("Given Namespace and Secret Name not found") - } - // Use the secret as needed - // for _, key := range rootToken { - // fmt.Println("Root Token", key) - // } - // Mount the unseal key and root token to the Vault path - - return rootToken,values, nil -} \ No newline at end of file diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 1b2c785b..99ca888e 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -30,39 +30,39 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") - vc, err := client.NewVaultClient(v.log, v.conf) - if err != nil { - v.log.Errorf("%s", err) - return - } - // addresses := []string{ - // v.conf.Address, - // v.conf.Address2, - // v.conf.Adddress3, + //vc, err := client.NewVaultClient(v.log, v.conf) + // if err != nil { + // v.log.Errorf("%s", err) + // return // } + addresses := []string{ + v.conf.Address, + v.conf.Address2, + v.conf.Adddress3, + } servicename := []string{"capten-dev-vault-0", "capten-dev-vault-1", "capten-dev-vault-2"} - // var vc *client.VaultClient - // var vaultClients []*client.VaultClient - // for _, address := range addresses { - // conf := config.VaultEnv{ - // Address: address, - // ReadTimeout: 30, - // MaxRetries: 3, + var vc *client.VaultClient + var vaultClients []*client.VaultClient + for _, address := range addresses { + conf := config.VaultEnv{ + Address: address, + ReadTimeout: 30, + MaxRetries: 3, - // // Set other configuration options as needed - // } - // v.log.Debug("Address Configuration", conf) + // Set other configuration options as needed + } + v.log.Debug("Address Configuration", conf) - // vc, err := client.NewVaultClient(v.log, v.conf) + vc, err := client.NewVaultClient(v.log, v.conf) - // if err != nil { - // v.log.Errorf("%s", err) - // return - // } + if err != nil { + v.log.Errorf("%s", err) + return + } - // vaultClients = append(vaultClients, vc) - // } + vaultClients = append(vaultClients, vc) + } // v.log.Debug("Vault Clients", vaultClients) if v.conf.HAEnabled { @@ -70,19 +70,19 @@ func (v *VaultSealWatcher) Run() { v.log.Infof("HA ENABLED", v.conf.HAEnabled) for _, svc := range servicename { - // switch svc { - // case "capten-dev-vault-0": - // vc = vaultClients[0] - // v.log.Debug("Vault Client:", vc) - // case "capten-dev-vault-1": - // vc = vaultClients[1] - // v.log.Debug("Vault Client:", vc) - // case "capten-dev-vault-2": - // vc = vaultClients[2] - // v.log.Debug("Vault Client:", vc) - // default: - // // Handle the case where the service name doesn't match any of the instances - // } + switch svc { + case "capten-dev-vault-0": + vc = vaultClients[0] + + case "capten-dev-vault-1": + vc = vaultClients[1] + + case "capten-dev-vault-2": + vc = vaultClients[2] + + default: + // Handle the case where the service name doesn't match any of the instances + } podip, err := vc.GetPodIP(svc, "platform") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) @@ -96,7 +96,7 @@ func (v *VaultSealWatcher) Run() { if res { v.log.Info("vault is sealed, trying to unseal") if svc == "capten-dev-vault-0" { - _, unsealKeys, err := vc.RetrieveKeys("platform", v.conf.VaultSecretName) + _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() v.log.Debug("Unseal Keys", unsealKeys) if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) @@ -120,7 +120,7 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to retrieve pod ip, %s", err) return } - _, unsealKeys, err := vc.RetrieveKeys("platform", v.conf.VaultSecretName) + _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() v.log.Debug("Unseal Keys", unsealKeys) if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) From c84c2d4615e40d3f42d164b3ff31689ac1be5a2b Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 15 Sep 2023 12:36:56 +0530 Subject: [PATCH 30/64] Modified code to test devcluster --- internal/job/vault_seal_watcher.go | 86 +++++++++++++++--------------- server/server.go | 46 ++++++++-------- 2 files changed, 67 insertions(+), 65 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 99ca888e..aab3abda 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -30,39 +30,39 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") - //vc, err := client.NewVaultClient(v.log, v.conf) - // if err != nil { - // v.log.Errorf("%s", err) - // return - // } - addresses := []string{ - v.conf.Address, - v.conf.Address2, - v.conf.Adddress3, + vc, err := client.NewVaultClient(v.log, v.conf) + if err != nil { + v.log.Errorf("%s", err) + return } - servicename := []string{"capten-dev-vault-0", "capten-dev-vault-1", "capten-dev-vault-2"} + // addresses := []string{ + // v.conf.Address, + // v.conf.Address2, + // v.conf.Adddress3, + // } + servicename := []string{"vault-hash-0", "vault-hash-1", "vault-hash-2"} - var vc *client.VaultClient - var vaultClients []*client.VaultClient - for _, address := range addresses { - conf := config.VaultEnv{ - Address: address, - ReadTimeout: 30, - MaxRetries: 3, + // var vc *client.VaultClient + // var vaultClients []*client.VaultClient + // for _, address := range addresses { + // conf := config.VaultEnv{ + // Address: address, + // ReadTimeout: 30, + // MaxRetries: 3, - // Set other configuration options as needed - } - v.log.Debug("Address Configuration", conf) + // // Set other configuration options as needed + // } + // v.log.Debug("Address Configuration", conf) - vc, err := client.NewVaultClient(v.log, v.conf) + // vc, err := client.NewVaultClient(v.log, v.conf) - if err != nil { - v.log.Errorf("%s", err) - return - } + // if err != nil { + // v.log.Errorf("%s", err) + // return + // } - vaultClients = append(vaultClients, vc) - } + // vaultClients = append(vaultClients, vc) + // } // v.log.Debug("Vault Clients", vaultClients) if v.conf.HAEnabled { @@ -70,39 +70,41 @@ func (v *VaultSealWatcher) Run() { v.log.Infof("HA ENABLED", v.conf.HAEnabled) for _, svc := range servicename { - switch svc { - case "capten-dev-vault-0": - vc = vaultClients[0] + // switch svc { + // case "capten-dev-vault-0": + // vc = vaultClients[0] - case "capten-dev-vault-1": - vc = vaultClients[1] + // case "capten-dev-vault-1": + // vc = vaultClients[1] - case "capten-dev-vault-2": - vc = vaultClients[2] + // case "capten-dev-vault-2": + // vc = vaultClients[2] - default: - // Handle the case where the service name doesn't match any of the instances - } - podip, err := vc.GetPodIP(svc, "platform") + // default: + // // Handle the case where the service name doesn't match any of the instances + // } + podip, err := vc.GetPodIP(svc, "default") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return } + v.log.Info("POD IP",podip) res, err := vc.IsVaultSealedForAllInstances(podip) if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) return } + v.log.Info("Seal Status",res) if res { v.log.Info("vault is sealed, trying to unseal") - if svc == "capten-dev-vault-0" { + if svc == "vault-hash-0" { _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() v.log.Debug("Unseal Keys", unsealKeys) if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - podip, err := vc.GetPodIP(svc, "platform") + podip, err := vc.GetPodIP(svc, "default") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return @@ -115,7 +117,7 @@ func (v *VaultSealWatcher) Run() { } } else { - podip, err := vc.GetPodIP(svc, "platform") + podip, err := vc.GetPodIP(svc, "default") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return @@ -175,7 +177,7 @@ func (v *VaultSealWatcher) Run() { } for _, svc := range servicename { - podip, _ := vc.GetPodIP(svc, "platform") + podip, _ := vc.GetPodIP(svc, "default") res, err := vc.IsVaultSealedForAllInstances(podip) v.log.Debug("Seal Status of %v :%v", svc, res) diff --git a/server/server.go b/server/server.go index 685f4cec..a1c065e1 100644 --- a/server/server.go +++ b/server/server.go @@ -75,28 +75,28 @@ func initScheduler(log logging.Logger, cfg config.Configuration) (s *job.Schedul } } - if cfg.VaultPolicyWatchInterval != "" { - pj, err := job.NewVaultPolicyWatcher(log, cfg.VaultPolicyWatchInterval) - if err != nil { - log.Fatal("failed to init policy watcher job", err) - } - - err = s.AddJob("vault-policy-watcher", pj) - if err != nil { - log.Fatal("failed to add policy watcher job", err) - } - } - - if cfg.VaultCredSyncInterval != "" { - pj, err := job.NewVaultCredSync(log, cfg.VaultCredSyncInterval) - if err != nil { - log.Fatal("failed to init cred sync job", err) - } - - err = s.AddJob("vault-cred-sync", pj) - if err != nil { - log.Fatal("failed to add cred sync job", err) - } - } + // if cfg.VaultPolicyWatchInterval != "" { + // pj, err := job.NewVaultPolicyWatcher(log, cfg.VaultPolicyWatchInterval) + // if err != nil { + // log.Fatal("failed to init policy watcher job", err) + // } + + // err = s.AddJob("vault-policy-watcher", pj) + // if err != nil { + // log.Fatal("failed to add policy watcher job", err) + // } + // } + + // if cfg.VaultCredSyncInterval != "" { + // pj, err := job.NewVaultCredSync(log, cfg.VaultCredSyncInterval) + // if err != nil { + // log.Fatal("failed to init cred sync job", err) + // } + + // err = s.AddJob("vault-cred-sync", pj) + // if err != nil { + // log.Fatal("failed to add cred sync job", err) + // } + // } return } From 1f01a1e25f561fe1f2b85eb8673d1cebddf99d18 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 15 Sep 2023 12:56:14 +0530 Subject: [PATCH 31/64] Modified code to test devcluster --- internal/client/vault_seal.go | 12 +++++++++--- internal/job/vault_seal_watcher.go | 20 ++++++++++---------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index febdca39..98a183c6 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -19,8 +19,13 @@ func (vc *VaultClient) IsVaultSealed() (bool, error) { return status.Sealed, nil } -func (vc *VaultClient) Unseal() error { - +func (vc *VaultClient) Unseal(podip string) error { + address := fmt.Sprintf("http://%s:8200", podip) + err := vc.c.SetAddress(address) + if err != nil { + vc.log.Errorf("Error while setting address") + } + vc.log.Debug("Address", address) status, err := vc.c.Sys().SealStatus() if err != nil { return err @@ -53,7 +58,8 @@ func (vc *VaultClient) Unseal() error { return nil } -func (vc *VaultClient) initializeVaultSecret() error { +func (vc *VaultClient) initializeVaultSecret(podip string) error { + unsealKeys, rootToken, err := vc.generateUnsealKeys() if err != nil { return errors.WithMessage(err, "error while generating unseal keys") diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index aab3abda..ad9a5e7a 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -88,29 +88,29 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to retrieve pod ip, %s", err) return } - v.log.Info("POD IP",podip) + v.log.Info("POD IP", podip) res, err := vc.IsVaultSealedForAllInstances(podip) if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) return } - v.log.Info("Seal Status",res) + v.log.Info("Seal Status", res) if res { v.log.Info("vault is sealed, trying to unseal") if svc == "vault-hash-0" { - _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - v.log.Debug("Unseal Keys", unsealKeys) - if err != nil { - v.log.Errorf("Failed to fetch the credential: %v\n", err) - return - } + // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + // v.log.Debug("Unseal Keys", unsealKeys) + // if err != nil { + // v.log.Errorf("Failed to fetch the credential: %v\n", err) + // return + // } podip, err := vc.GetPodIP(svc, "default") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return } - // err := vc.Unseal() - err = vc.UnsealVaultInstance(podip, unsealKeys) + err = vc.Unseal(podip) + // err = vc.UnsealVaultInstance(podip, unsealKeys) if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return From be9df0e15ef3174ea9a5d2d8a4b3ac154c47b9ab Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 15 Sep 2023 12:57:44 +0530 Subject: [PATCH 32/64] Modified code to test devcluster --- internal/client/vault_seal.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 98a183c6..2e5f2fb9 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -20,7 +20,7 @@ func (vc *VaultClient) IsVaultSealed() (bool, error) { } func (vc *VaultClient) Unseal(podip string) error { - address := fmt.Sprintf("http://%s:8200", podip) + address := fmt.Sprintf("http://%s:8200", podip) err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") @@ -58,7 +58,7 @@ func (vc *VaultClient) Unseal(podip string) error { return nil } -func (vc *VaultClient) initializeVaultSecret(podip string) error { +func (vc *VaultClient) initializeVaultSecret() error { unsealKeys, rootToken, err := vc.generateUnsealKeys() if err != nil { @@ -236,13 +236,3 @@ func (vc *VaultClient) GetPodIP(podName, namespace string) (string, error) { vc.log.Debug("Pod ip", pod.Status.PodIP) return pod.Status.PodIP, nil } - - - - - - - - - - From b9cc5ea07dfe4739e5dc23f8f843006b4a5893ea Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 15 Sep 2023 13:06:25 +0530 Subject: [PATCH 33/64] Modified code to test devcluster --- internal/client/vault_seal.go | 10 +++------- internal/job/vault_seal_watcher.go | 12 ++++++------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 2e5f2fb9..7e96b777 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -19,13 +19,9 @@ func (vc *VaultClient) IsVaultSealed() (bool, error) { return status.Sealed, nil } -func (vc *VaultClient) Unseal(podip string) error { - address := fmt.Sprintf("http://%s:8200", podip) - err := vc.c.SetAddress(address) - if err != nil { - vc.log.Errorf("Error while setting address") - } - vc.log.Debug("Address", address) +func (vc *VaultClient) Unseal() error { + + status, err := vc.c.Sys().SealStatus() if err != nil { return err diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index ad9a5e7a..4312357f 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -104,12 +104,12 @@ func (v *VaultSealWatcher) Run() { // v.log.Errorf("Failed to fetch the credential: %v\n", err) // return // } - podip, err := vc.GetPodIP(svc, "default") - if err != nil { - v.log.Errorf("failed to retrieve pod ip, %s", err) - return - } - err = vc.Unseal(podip) + // podip, err := vc.GetPodIP(svc, "default") + // if err != nil { + // v.log.Errorf("failed to retrieve pod ip, %s", err) + // return + // } + err = vc.Unseal() // err = vc.UnsealVaultInstance(podip, unsealKeys) if err != nil { v.log.Errorf("failed to unseal vault, %s", err) From 3f86e36a625898616f6b841a4dce4cbdcc4d590b Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 15 Sep 2023 13:39:29 +0530 Subject: [PATCH 34/64] Modified code to test devcluster --- config/config.go | 2 +- internal/job/vault_seal_watcher.go | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index 7728a387..06368c77 100644 --- a/config/config.go +++ b/config/config.go @@ -16,7 +16,7 @@ type Configuration struct { type VaultEnv struct { HAEnabled bool `envconfig:"HA_ENABLED" default:"true"` - Address string `envconfig:"VAULT_ADDR" default:"http://capten-dev-vault-0:8200"` + Address string `envconfig:"VAULT_ADDR" default:"http://vault-hash:8200"` Address2 string `envconfig:"VAULT_ADDR2" default:"http://capten-dev-vault-1:8200"` Adddress3 string `envconfig:"VAULT_ADDR3" default:"http://capten-dev-vault-2:8200"` CACert string `envconfig:"VAULT_CACERT" required:"false"` diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 4312357f..a0b82c75 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -109,6 +109,7 @@ func (v *VaultSealWatcher) Run() { // v.log.Errorf("failed to retrieve pod ip, %s", err) // return // } + v.log.Info("Unsealing for first instance") err = vc.Unseal() // err = vc.UnsealVaultInstance(podip, unsealKeys) if err != nil { @@ -129,10 +130,10 @@ func (v *VaultSealWatcher) Run() { return } //podip,err:=vc.GetPodIP(svc,"platform") - if err != nil { - v.log.Errorf("failed to retrieve pod ip, %s", err) - return - } + // if err != nil { + // v.log.Errorf("failed to retrieve pod ip, %s", err) + // return + // } err = vc.UnsealVaultInstance(podip, unsealKeys) //err = vc.Unseal() if err != nil { From 98121a684e8fca5b581f512f095cb3deff6a15d1 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 15 Sep 2023 15:05:55 +0530 Subject: [PATCH 35/64] Modified code to test devcluster --- internal/client/vault_seal.go | 2 +- internal/job/vault_seal_watcher.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 7e96b777..7593c27c 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -21,7 +21,7 @@ func (vc *VaultClient) IsVaultSealed() (bool, error) { func (vc *VaultClient) Unseal() error { - + vc.log.Info("Unsealing for first instance inside unseal func") status, err := vc.c.Sys().SealStatus() if err != nil { return err diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index a0b82c75..de7e0810 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -118,7 +118,9 @@ func (v *VaultSealWatcher) Run() { } } else { + podip, err := vc.GetPodIP(svc, "default") + v.log.Info("Unsealing for second % vinstance", podip) if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return From ff83f1fcf9f163881e0b26b72aeba12ce50cc491 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 15 Sep 2023 15:25:50 +0530 Subject: [PATCH 36/64] Modified code to test devcluster --- charts/vault-cred/values.yaml | 2 +- internal/client/vault_seal.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/charts/vault-cred/values.yaml b/charts/vault-cred/values.yaml index 5e8d730a..da8183b2 100644 --- a/charts/vault-cred/values.yaml +++ b/charts/vault-cred/values.yaml @@ -45,7 +45,7 @@ env: vault: haenabled: true - vaultAddress: http://capten-dev-vault-0:8200 + vaultAddress: http://vault-hash:8200 secretName: vault-server secretTokenKeyName: roottoken secretUnSealKeyPrefix: unsealkey diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 7593c27c..3a114187 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -30,12 +30,14 @@ func (vc *VaultClient) Unseal() error { if !status.Sealed { return nil } + vc.log.Info("Status",status) rootToken, unsealKeys, err := vc.getVaultSecretValues() if err != nil { return err } - + vc.log.Info("Root Token",rootToken) + vc.log.Info("Unseal Keys",unsealKeys) if !status.Initialized && len(rootToken) == 0 && len(unsealKeys) == 0 { vc.log.Debug("intializing vault secret") err = vc.initializeVaultSecret() @@ -57,6 +59,8 @@ func (vc *VaultClient) Unseal() error { func (vc *VaultClient) initializeVaultSecret() error { unsealKeys, rootToken, err := vc.generateUnsealKeys() + vc.log.Info("Unseal Keys",unsealKeys) + vc.log.Info("Root token",rootToken) if err != nil { return errors.WithMessage(err, "error while generating unseal keys") } From 98261f29750a0aff49b0683525166391e1ca5a31 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 15 Sep 2023 18:26:39 +0530 Subject: [PATCH 37/64] Modified code to test devcluster --- internal/job/vault_seal_watcher.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index de7e0810..9c6eb242 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -125,6 +125,12 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to retrieve pod ip, %s", err) return } + err = vc.JoinRaftCluster(podip) + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return + + } _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() v.log.Debug("Unseal Keys", unsealKeys) if err != nil { @@ -143,12 +149,7 @@ func (v *VaultSealWatcher) Run() { return } // v.log.Info("vault unsealed executed") - err = vc.JoinRaftCluster(podip) - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - return - } } // err := vc.Unseal() From 5fc2621991766244093f373f6e0bce6ce957aed0 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 15 Sep 2023 23:08:13 +0530 Subject: [PATCH 38/64] Modified code to test devcluster --- config/config.go | 6 +- internal/job/vault_seal_watcher.go | 135 +++++++++++------------------ server/server.go | 46 +++++----- 3 files changed, 75 insertions(+), 112 deletions(-) diff --git a/config/config.go b/config/config.go index 06368c77..8e0ba695 100644 --- a/config/config.go +++ b/config/config.go @@ -16,9 +16,9 @@ type Configuration struct { type VaultEnv struct { HAEnabled bool `envconfig:"HA_ENABLED" default:"true"` - Address string `envconfig:"VAULT_ADDR" default:"http://vault-hash:8200"` - Address2 string `envconfig:"VAULT_ADDR2" default:"http://capten-dev-vault-1:8200"` - Adddress3 string `envconfig:"VAULT_ADDR3" default:"http://capten-dev-vault-2:8200"` + Address string `envconfig:"VAULT_ADDR" default:"http://vault-hash-0:8200"` + Address2 string `envconfig:"VAULT_ADDR2" default:"http://vault-hash-1:8200"` + Adddress3 string `envconfig:"VAULT_ADDR3" default:"http://vault-hash-2:8200"` CACert string `envconfig:"VAULT_CACERT" required:"false"` ReadTimeout time.Duration `envconfig:"VAULT_READ_TIMEOUT" default:"60s"` MaxRetries int `envconfig:"VAULT_MAX_RETRIES" default:"5"` diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 9c6eb242..9d6ac3d9 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -30,59 +30,59 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") - vc, err := client.NewVaultClient(v.log, v.conf) - if err != nil { - v.log.Errorf("%s", err) - return - } - // addresses := []string{ - // v.conf.Address, - // v.conf.Address2, - // v.conf.Adddress3, +// vc, err := client.NewVaultClient(v.log, v.conf) + // if err != nil { + // v.log.Errorf("%s", err) + // return // } + addresses := []string{ + v.conf.Address, + v.conf.Address2, + v.conf.Adddress3, + } servicename := []string{"vault-hash-0", "vault-hash-1", "vault-hash-2"} - // var vc *client.VaultClient - // var vaultClients []*client.VaultClient - // for _, address := range addresses { - // conf := config.VaultEnv{ - // Address: address, - // ReadTimeout: 30, - // MaxRetries: 3, + var vc *client.VaultClient + var vaultClients []*client.VaultClient + for _, address := range addresses { + conf := config.VaultEnv{ + Address: address, + ReadTimeout: 30, + MaxRetries: 3, - // // Set other configuration options as needed - // } - // v.log.Debug("Address Configuration", conf) + + } + v.log.Debug("Address Configuration", conf) - // vc, err := client.NewVaultClient(v.log, v.conf) + vc, err := client.NewVaultClient(v.log, v.conf) - // if err != nil { - // v.log.Errorf("%s", err) - // return - // } + if err != nil { + v.log.Errorf("%s", err) + return + } - // vaultClients = append(vaultClients, vc) - // } - // v.log.Debug("Vault Clients", vaultClients) + vaultClients = append(vaultClients, vc) + } + v.log.Debug("Vault Clients", vaultClients) if v.conf.HAEnabled { v.log.Infof("HA ENABLED", v.conf.HAEnabled) for _, svc := range servicename { - // switch svc { - // case "capten-dev-vault-0": - // vc = vaultClients[0] + switch svc { + case "vault-hash-0": + vc = vaultClients[0] - // case "capten-dev-vault-1": - // vc = vaultClients[1] + case "vault-hash-1": + vc = vaultClients[1] - // case "capten-dev-vault-2": - // vc = vaultClients[2] + case "vault-hash-2": + vc = vaultClients[2] - // default: - // // Handle the case where the service name doesn't match any of the instances - // } + default: + // Handle the case where the service name doesn't match any of the instances + } podip, err := vc.GetPodIP(svc, "default") if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) @@ -98,20 +98,10 @@ func (v *VaultSealWatcher) Run() { if res { v.log.Info("vault is sealed, trying to unseal") if svc == "vault-hash-0" { - // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - // v.log.Debug("Unseal Keys", unsealKeys) - // if err != nil { - // v.log.Errorf("Failed to fetch the credential: %v\n", err) - // return - // } - // podip, err := vc.GetPodIP(svc, "default") - // if err != nil { - // v.log.Errorf("failed to retrieve pod ip, %s", err) - // return - // } + v.log.Info("Unsealing for first instance") err = vc.Unseal() - // err = vc.UnsealVaultInstance(podip, unsealKeys) + if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return @@ -125,58 +115,31 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to retrieve pod ip, %s", err) return } - err = vc.JoinRaftCluster(podip) - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - return - } _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() v.log.Debug("Unseal Keys", unsealKeys) if err != nil { v.log.Errorf("Failed to fetch the credential: %v\n", err) return } - //podip,err:=vc.GetPodIP(svc,"platform") - // if err != nil { - // v.log.Errorf("failed to retrieve pod ip, %s", err) - // return - // } + err = vc.UnsealVaultInstance(podip, unsealKeys) - //err = vc.Unseal() + if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return } - // v.log.Info("vault unsealed executed") + err = vc.JoinRaftCluster(podip) + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return + + } + } - // err := vc.Unseal() - // if err != nil { - // v.log.Errorf("failed to unseal vault, %s", err) - // return - // } - - // res, err := vc.IsVaultSealed() - // if res { - - // err := vc.Unseal() - // if err != nil { - // v.log.Errorf("failed to unseal vault, %s", err) - // return - // } - - // } - // if err != nil { - // v.log.Errorf("failed to get vault seal status, %s", err) - // return - // } - // v.log.Infof("vault sealed status: %v", res) - - // } else { - // v.log.Debug("vault is in unsealed status") - // } + } } diff --git a/server/server.go b/server/server.go index a1c065e1..685f4cec 100644 --- a/server/server.go +++ b/server/server.go @@ -75,28 +75,28 @@ func initScheduler(log logging.Logger, cfg config.Configuration) (s *job.Schedul } } - // if cfg.VaultPolicyWatchInterval != "" { - // pj, err := job.NewVaultPolicyWatcher(log, cfg.VaultPolicyWatchInterval) - // if err != nil { - // log.Fatal("failed to init policy watcher job", err) - // } - - // err = s.AddJob("vault-policy-watcher", pj) - // if err != nil { - // log.Fatal("failed to add policy watcher job", err) - // } - // } - - // if cfg.VaultCredSyncInterval != "" { - // pj, err := job.NewVaultCredSync(log, cfg.VaultCredSyncInterval) - // if err != nil { - // log.Fatal("failed to init cred sync job", err) - // } - - // err = s.AddJob("vault-cred-sync", pj) - // if err != nil { - // log.Fatal("failed to add cred sync job", err) - // } - // } + if cfg.VaultPolicyWatchInterval != "" { + pj, err := job.NewVaultPolicyWatcher(log, cfg.VaultPolicyWatchInterval) + if err != nil { + log.Fatal("failed to init policy watcher job", err) + } + + err = s.AddJob("vault-policy-watcher", pj) + if err != nil { + log.Fatal("failed to add policy watcher job", err) + } + } + + if cfg.VaultCredSyncInterval != "" { + pj, err := job.NewVaultCredSync(log, cfg.VaultCredSyncInterval) + if err != nil { + log.Fatal("failed to init cred sync job", err) + } + + err = s.AddJob("vault-cred-sync", pj) + if err != nil { + log.Fatal("failed to add cred sync job", err) + } + } return } From 5f9044e14d67c9ba1431a13a976a8e73716b2dc4 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 18 Sep 2023 13:13:03 +0530 Subject: [PATCH 39/64] Modified vault unseal --- internal/job/vault_seal_watcher.go | 12 ++++++-- server/server.go | 46 +++++++++++++++--------------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 9d6ac3d9..4d39c838 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -73,13 +73,14 @@ func (v *VaultSealWatcher) Run() { switch svc { case "vault-hash-0": vc = vaultClients[0] + v.log.Debug("Vault Client",vc) case "vault-hash-1": vc = vaultClients[1] - + v.log.Debug("Vault Client",vc) case "vault-hash-2": vc = vaultClients[2] - + v.log.Debug("Vault Client",vc) default: // Handle the case where the service name doesn't match any of the instances } @@ -100,7 +101,12 @@ func (v *VaultSealWatcher) Run() { if svc == "vault-hash-0" { v.log.Info("Unsealing for first instance") - err = vc.Unseal() + _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + if err != nil { + v.log.Errorf("Failed to fetch the credential: %v\n", err) + return + } + err = vc.UnsealVaultInstance(podip,unsealKeys) if err != nil { v.log.Errorf("failed to unseal vault, %s", err) diff --git a/server/server.go b/server/server.go index 685f4cec..a1c065e1 100644 --- a/server/server.go +++ b/server/server.go @@ -75,28 +75,28 @@ func initScheduler(log logging.Logger, cfg config.Configuration) (s *job.Schedul } } - if cfg.VaultPolicyWatchInterval != "" { - pj, err := job.NewVaultPolicyWatcher(log, cfg.VaultPolicyWatchInterval) - if err != nil { - log.Fatal("failed to init policy watcher job", err) - } - - err = s.AddJob("vault-policy-watcher", pj) - if err != nil { - log.Fatal("failed to add policy watcher job", err) - } - } - - if cfg.VaultCredSyncInterval != "" { - pj, err := job.NewVaultCredSync(log, cfg.VaultCredSyncInterval) - if err != nil { - log.Fatal("failed to init cred sync job", err) - } - - err = s.AddJob("vault-cred-sync", pj) - if err != nil { - log.Fatal("failed to add cred sync job", err) - } - } + // if cfg.VaultPolicyWatchInterval != "" { + // pj, err := job.NewVaultPolicyWatcher(log, cfg.VaultPolicyWatchInterval) + // if err != nil { + // log.Fatal("failed to init policy watcher job", err) + // } + + // err = s.AddJob("vault-policy-watcher", pj) + // if err != nil { + // log.Fatal("failed to add policy watcher job", err) + // } + // } + + // if cfg.VaultCredSyncInterval != "" { + // pj, err := job.NewVaultCredSync(log, cfg.VaultCredSyncInterval) + // if err != nil { + // log.Fatal("failed to init cred sync job", err) + // } + + // err = s.AddJob("vault-cred-sync", pj) + // if err != nil { + // log.Fatal("failed to add cred sync job", err) + // } + // } return } From 64df23c140ed036cb8ac444aa0da17f037eecd40 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 18 Sep 2023 13:28:39 +0530 Subject: [PATCH 40/64] Modified vault unseal --- config/config.go | 2 +- internal/job/vault_seal_watcher.go | 28 ++++++++++++---------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/config/config.go b/config/config.go index 8e0ba695..6c82efa9 100644 --- a/config/config.go +++ b/config/config.go @@ -24,7 +24,7 @@ type VaultEnv struct { MaxRetries int `envconfig:"VAULT_MAX_RETRIES" default:"5"` VaultTokenForRequests bool `envconfig:"VAULT_TOKEN_FOR_REQUESTS" default:"false"` VaultSecretName string `envconfig:"VAULT_SECRET_NAME" default:"vault-server"` - VaultSecretNameSpace string `envconfig:"POD_NAMESPACE" required:"true"` + VaultSecretNameSpace string `envconfig:"POD_NAMESPACE" default:"default" required:"true"` VaultSecretTokenKeyName string `envconfig:"VAULT_SECRET_TOKEN_KEY_NAME" default:"root-token"` VaultSecretUnSealKeyPrefix string `envconfig:"VAULT_SECRET_UNSEAL_KEY_PREFIX" default:"unsealkey"` VaultToken string `envconfig:"VAULT_TOKEN"` diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 4d39c838..e8a74151 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -30,7 +30,7 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") -// vc, err := client.NewVaultClient(v.log, v.conf) + // vc, err := client.NewVaultClient(v.log, v.conf) // if err != nil { // v.log.Errorf("%s", err) // return @@ -49,8 +49,6 @@ func (v *VaultSealWatcher) Run() { Address: address, ReadTimeout: 30, MaxRetries: 3, - - } v.log.Debug("Address Configuration", conf) @@ -73,14 +71,14 @@ func (v *VaultSealWatcher) Run() { switch svc { case "vault-hash-0": vc = vaultClients[0] - v.log.Debug("Vault Client",vc) + v.log.Debug("Vault Client", vc) case "vault-hash-1": vc = vaultClients[1] - v.log.Debug("Vault Client",vc) + v.log.Debug("Vault Client", vc) case "vault-hash-2": vc = vaultClients[2] - v.log.Debug("Vault Client",vc) + v.log.Debug("Vault Client", vc) default: // Handle the case where the service name doesn't match any of the instances } @@ -99,15 +97,15 @@ func (v *VaultSealWatcher) Run() { if res { v.log.Info("vault is sealed, trying to unseal") if svc == "vault-hash-0" { - + v.log.Info("Unsealing for first instance") - _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - if err != nil { - v.log.Errorf("Failed to fetch the credential: %v\n", err) - return - } - err = vc.UnsealVaultInstance(podip,unsealKeys) - + // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + // if err != nil { + // v.log.Errorf("Failed to fetch the credential: %v\n", err) + // return + // } + //err = vc.UnsealVaultInstance(podip,unsealKeys) + err := vc.Unseal() if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return @@ -141,11 +139,9 @@ func (v *VaultSealWatcher) Run() { return } - } - } } From 62b5aac8e986074d66dd9a0915c2b381601e9d94 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 18 Sep 2023 14:13:43 +0530 Subject: [PATCH 41/64] Modified vault unseal --- internal/client/vault.go | 4 +++- internal/job/vault_seal_watcher.go | 11 +++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index c9d5fef9..08f67338 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -181,6 +181,7 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa func (vc *VaultClient) JoinRaftCluster(podip string) error { var req *api.RaftJoinRequest address := fmt.Sprintf("http://%s:8200", podip) + err := vc.c.SetAddress(address) if err != nil { vc.log.Errorf("Error while setting address") @@ -188,10 +189,11 @@ func (vc *VaultClient) JoinRaftCluster(podip string) error { vc.log.Debug("Address", address) leaderInfo, err := vc.c.Sys().Leader() + vc.log.Debug("Leader address",leaderInfo.LeaderAddress) if err != nil { - vc.log.Debugf("Failed to retrieve leader information: %v", err) return err + } if leaderInfo.LeaderAddress == "" { diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index e8a74151..4abc67ed 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -119,7 +119,12 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to retrieve pod ip, %s", err) return } + err = vc.JoinRaftCluster(podip) + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return + } _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() v.log.Debug("Unseal Keys", unsealKeys) if err != nil { @@ -133,12 +138,6 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to unseal vault, %s", err) return } - err = vc.JoinRaftCluster(podip) - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - return - - } } From eef9552322e99372783b6b9609d90511e3bdeeaf Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 18 Sep 2023 15:24:07 +0530 Subject: [PATCH 42/64] Modified vault unseal --- internal/client/vault.go | 43 ++++++++++++++---------------- internal/job/vault_seal_watcher.go | 15 ++++++----- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index 08f67338..4c95a5e0 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -178,48 +178,45 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa return } + + func (vc *VaultClient) JoinRaftCluster(podip string) error { - var req *api.RaftJoinRequest + // Construct the Vault API address address := fmt.Sprintf("http://%s:8200", podip) - err := vc.c.SetAddress(address) - if err != nil { - vc.log.Errorf("Error while setting address") + // Set the Vault client address + if err := vc.c.SetAddress(address); err != nil { + return fmt.Errorf("failed to set Vault client address: %v", err) } - vc.log.Debug("Address", address) + vc.log.Debugf("Address: %s", address) + + // Retrieve leader information leaderInfo, err := vc.c.Sys().Leader() - vc.log.Debug("Leader address",leaderInfo.LeaderAddress) if err != nil { - vc.log.Debugf("Failed to retrieve leader information: %v", err) - return err - + return fmt.Errorf("failed to retrieve leader information: %v", err) } + vc.log.Debugf("Leader address: %s", leaderInfo.LeaderAddress) + if leaderInfo.LeaderAddress == "" { // Handle the case where leader address is empty vc.log.Debug("Leader address is empty") - return err - } else { - req = &api.RaftJoinRequest{ - Retry: true, - LeaderAPIAddr: leaderInfo.LeaderAddress, - } + return fmt.Errorf("leader address is empty") } - // req := &api.RaftJoinRequest{ - // Retry: true, - // LeaderAPIAddr: leaderInfo.LeaderAddress, - // } + req := &api.RaftJoinRequest{ + Retry: true, + LeaderAPIAddr: leaderInfo.LeaderAddress, + } vc.log.Debugf("Leader API address: %s", leaderInfo.LeaderAddress) - _, err = vc.c.Sys().RaftJoin(req) // Replace with your leader address + _, err = vc.c.Sys().RaftJoin(req) if err != nil { - // Handle the join error (e.g., return it or log and return) - vc.log.Debugf("Failed to join the Raft cluster: %v", err) - return err + return fmt.Errorf("failed to join the Raft cluster: %v", err) } return nil } + diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 4abc67ed..235f4534 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -45,14 +45,16 @@ func (v *VaultSealWatcher) Run() { var vc *client.VaultClient var vaultClients []*client.VaultClient for _, address := range addresses { - conf := config.VaultEnv{ - Address: address, - ReadTimeout: 30, - MaxRetries: 3, - } + conf := v.conf // Make a copy of the existing configuration + conf.Address = address + // conf := config.VaultEnv{ + // Address: address, + // // ReadTimeout: 30, + // // MaxRetries: 3, + // } v.log.Debug("Address Configuration", conf) - vc, err := client.NewVaultClient(v.log, v.conf) + vc, err := client.NewVaultClient(v.log, conf) if err != nil { v.log.Errorf("%s", err) @@ -119,6 +121,7 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to retrieve pod ip, %s", err) return } + v.log.Debug("POD IP", podip) err = vc.JoinRaftCluster(podip) if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) From cf3748ad496236d4b019a76276b048c02a98318c Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 18 Sep 2023 15:42:22 +0530 Subject: [PATCH 43/64] Modified vault unseal --- internal/client/vault.go | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index 4c95a5e0..d8553624 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -178,45 +178,44 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa return } - - func (vc *VaultClient) JoinRaftCluster(podip string) error { // Construct the Vault API address address := fmt.Sprintf("http://%s:8200", podip) // Set the Vault client address if err := vc.c.SetAddress(address); err != nil { - return fmt.Errorf("failed to set Vault client address: %v", err) + vc.log.Debug("failed to set Vault client address: %v", err) + return err } vc.log.Debugf("Address: %s", address) // Retrieve leader information - leaderInfo, err := vc.c.Sys().Leader() - if err != nil { - return fmt.Errorf("failed to retrieve leader information: %v", err) - } + // leaderInfo, err := vc.c.Sys().Leader() + // if err != nil { + // return fmt.Errorf("failed to retrieve leader information: %v", err) + // } - vc.log.Debugf("Leader address: %s", leaderInfo.LeaderAddress) + // vc.log.Debugf("Leader address: %s", leaderInfo.LeaderAddress) - if leaderInfo.LeaderAddress == "" { - // Handle the case where leader address is empty - vc.log.Debug("Leader address is empty") - return fmt.Errorf("leader address is empty") - } + // if leaderInfo.LeaderAddress == "" { + // // Handle the case where leader address is empty + // vc.log.Debug("Leader address is empty") + // return fmt.Errorf("leader address is empty") + // } req := &api.RaftJoinRequest{ Retry: true, - LeaderAPIAddr: leaderInfo.LeaderAddress, + LeaderAPIAddr: "http://vault-hash-0.vault-hash-internal:8200", + //LeaderAPIAddr: leaderInfo.LeaderAddress, } - vc.log.Debugf("Leader API address: %s", leaderInfo.LeaderAddress) + //vc.log.Debugf("Leader API address: %s", leaderInfo.LeaderAddress) - _, err = vc.c.Sys().RaftJoin(req) + _, err := vc.c.Sys().RaftJoin(req) if err != nil { return fmt.Errorf("failed to join the Raft cluster: %v", err) } return nil } - From 40955f8e373dbf0f021cd2d49f1302f81de87a15 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 18 Sep 2023 17:33:30 +0530 Subject: [PATCH 44/64] Modified vault unseal --- internal/client/k8s.go | 24 +++++++++++- internal/client/vault.go | 36 ++++++++++++++++-- internal/job/vault_seal_watcher.go | 59 +++++++++++++++++++++++------- 3 files changed, 101 insertions(+), 18 deletions(-) diff --git a/internal/client/k8s.go b/internal/client/k8s.go index a0a0f5be..2fca2035 100644 --- a/internal/client/k8s.go +++ b/internal/client/k8s.go @@ -13,7 +13,9 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) - +const ( + labelSelector = "app.kubernetes.io/name=vault" +) type K8SClient struct { client *kubernetes.Clientset log logging.Logger @@ -132,3 +134,23 @@ func (k *K8SClient) GetConfigMapsHasPrefix(ctx context.Context, prefix string) ( } return allConfigMapData, nil } + +func (k *K8SClient) GetVaultPodInstances(ctx context.Context) ([]string, error) { + var podnames []string + + + pods, err := k.client.CoreV1().Pods("default").List(context.TODO(), metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return nil,errors.WithMessage(err, "error while retrieving the pods ") + } + + + for _, pod := range pods.Items { + podnames = append(podnames, pod.Name) + + + } + return podnames,nil +} \ No newline at end of file diff --git a/internal/client/vault.go b/internal/client/vault.go index d8553624..bf360151 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -178,7 +178,7 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa return } -func (vc *VaultClient) JoinRaftCluster(podip string) error { +func (vc *VaultClient) JoinRaftCluster(podip string, leaderaddress string) error { // Construct the Vault API address address := fmt.Sprintf("http://%s:8200", podip) @@ -190,13 +190,15 @@ func (vc *VaultClient) JoinRaftCluster(podip string) error { vc.log.Debugf("Address: %s", address) + // Extract the leader address from the response + // Retrieve leader information // leaderInfo, err := vc.c.Sys().Leader() // if err != nil { // return fmt.Errorf("failed to retrieve leader information: %v", err) // } - // vc.log.Debugf("Leader address: %s", leaderInfo.LeaderAddress) + //vc.log.Debugf("Leader address: %s", leaderInfo.LeaderAddress) // if leaderInfo.LeaderAddress == "" { // // Handle the case where leader address is empty @@ -206,7 +208,7 @@ func (vc *VaultClient) JoinRaftCluster(podip string) error { req := &api.RaftJoinRequest{ Retry: true, - LeaderAPIAddr: "http://vault-hash-0.vault-hash-internal:8200", + LeaderAPIAddr: leaderaddress, //LeaderAPIAddr: leaderInfo.LeaderAddress, } @@ -219,3 +221,31 @@ func (vc *VaultClient) JoinRaftCluster(podip string) error { return nil } + +func (vc *VaultClient) LeaderAPIAddr(podip string) (string, error) { + address := fmt.Sprintf("http://%s:8200", podip) + + if err := vc.c.SetAddress(address); err != nil { + vc.log.Debug("failed to set Vault client address: %v", err) + return "", err + } + + vc.log.Debugf("Address: %s", address) + + leaderInfo, err := vc.c.Sys().Leader() + if err != nil { + vc.log.Error("failed to retrieve leader information: %v", err) + return "", err + } + + vc.log.Debugf("Leader address: %s", leaderInfo.LeaderAddress) + + if leaderInfo.LeaderAddress == "" { + + vc.log.Debug("Leader address is empty") + } + leaderaddress := leaderInfo.LeaderAddress + + return leaderaddress, nil + +} diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 235f4534..53644015 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -1,6 +1,8 @@ package job import ( + "context" + "github.com/intelops/go-common/logging" "github.com/intelops/vault-cred/config" "github.com/intelops/vault-cred/internal/client" @@ -40,9 +42,20 @@ func (v *VaultSealWatcher) Run() { v.conf.Address2, v.conf.Adddress3, } - servicename := []string{"vault-hash-0", "vault-hash-1", "vault-hash-2"} + k8sclient, err := client.NewK8SClient(v.log) + if err != nil { + v.log.Errorf("Error while connecting with k8s %s", err) + return + } + podname, err := k8sclient.GetVaultPodInstances(context.Background()) + if err != nil { + v.log.Errorf("Error while retrieving vault instances %s", err) + return + } + //servicename := []string{"vault-hash-0", "vault-hash-1", "vault-hash-2"} var vc *client.VaultClient + var leaderpodip string var vaultClients []*client.VaultClient for _, address := range addresses { conf := v.conf // Make a copy of the existing configuration @@ -69,7 +82,7 @@ func (v *VaultSealWatcher) Run() { v.log.Infof("HA ENABLED", v.conf.HAEnabled) - for _, svc := range servicename { + for _, svc := range podname { switch svc { case "vault-hash-0": vc = vaultClients[0] @@ -84,7 +97,7 @@ func (v *VaultSealWatcher) Run() { default: // Handle the case where the service name doesn't match any of the instances } - podip, err := vc.GetPodIP(svc, "default") + podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return @@ -112,9 +125,22 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to unseal vault, %s", err) return } + podip, err := vc.GetPodIP(svc, "default") + v.log.Info("Unsealing for second % vinstance", podip) + if err != nil { + v.log.Errorf("failed to retrieve pod ip, %s", err) + return + } + leaderpodip = podip + v.log.Info("Leader Ip", leaderpodip) } else { - + leaderaddr, err := vc.LeaderAPIAddr(leaderpodip) + if err != nil { + v.log.Errorf("failed to retrieve leader address, %s", err) + return + } + v.log.Info("Leader Address", leaderaddr) podip, err := vc.GetPodIP(svc, "default") v.log.Info("Unsealing for second % vinstance", podip) if err != nil { @@ -122,32 +148,37 @@ func (v *VaultSealWatcher) Run() { return } v.log.Debug("POD IP", podip) - err = vc.JoinRaftCluster(podip) + err = vc.JoinRaftCluster(podip, leaderaddr) if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) return } - _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - v.log.Debug("Unseal Keys", unsealKeys) + err = vc.Unseal() if err != nil { - v.log.Errorf("Failed to fetch the credential: %v\n", err) + v.log.Errorf("failed to unseal vault, %s", err) return } + // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() + // v.log.Debug("Unseal Keys", unsealKeys) + // if err != nil { + // v.log.Errorf("Failed to fetch the credential: %v\n", err) + // return + // } - err = vc.UnsealVaultInstance(podip, unsealKeys) + // err = vc.UnsealVaultInstance(podip, unsealKeys) - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return - } + // if err != nil { + // v.log.Errorf("failed to unseal vault, %s", err) + // return + // } } } } - for _, svc := range servicename { + for _, svc := range podname { podip, _ := vc.GetPodIP(svc, "default") res, err := vc.IsVaultSealedForAllInstances(podip) From dac6b5b69befea8629cd4e0a113a66496aa90ba6 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 19 Sep 2023 10:46:18 +0530 Subject: [PATCH 45/64] Modified vault unseal --- internal/client/k8s.go | 5 ++++- internal/job/vault_seal_watcher.go | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/internal/client/k8s.go b/internal/client/k8s.go index 2fca2035..5c8e04a7 100644 --- a/internal/client/k8s.go +++ b/internal/client/k8s.go @@ -2,6 +2,7 @@ package client import ( "context" + "sort" "strings" "time" @@ -152,5 +153,7 @@ func (k *K8SClient) GetVaultPodInstances(ctx context.Context) ([]string, error) } + sort.Strings(podnames) + return podnames,nil -} \ No newline at end of file +} diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 53644015..e0f0720c 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -97,6 +97,7 @@ func (v *VaultSealWatcher) Run() { default: // Handle the case where the service name doesn't match any of the instances } + v.log.Info("Namespace", v.conf.VaultSecretNameSpace) podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) @@ -114,25 +115,25 @@ func (v *VaultSealWatcher) Run() { if svc == "vault-hash-0" { v.log.Info("Unsealing for first instance") + podip, err := vc.GetPodIP(svc, "default") + leaderpodip = podip + v.log.Info("Leader Ip", leaderpodip) + if err != nil { + v.log.Errorf("failed to retrieve pod ip, %s", err) + return + } + // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() // if err != nil { // v.log.Errorf("Failed to fetch the credential: %v\n", err) // return // } //err = vc.UnsealVaultInstance(podip,unsealKeys) - err := vc.Unseal() + err = vc.Unseal() if err != nil { v.log.Errorf("failed to unseal vault, %s", err) return } - podip, err := vc.GetPodIP(svc, "default") - v.log.Info("Unsealing for second % vinstance", podip) - if err != nil { - v.log.Errorf("failed to retrieve pod ip, %s", err) - return - } - leaderpodip = podip - v.log.Info("Leader Ip", leaderpodip) } else { leaderaddr, err := vc.LeaderAPIAddr(leaderpodip) From a57255006033d0cbf4046509ec4d3cce9b745ec7 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 19 Sep 2023 11:52:18 +0530 Subject: [PATCH 46/64] Modified vault unseal --- internal/job/vault_seal_watcher.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index e0f0720c..468d00f3 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -97,7 +97,7 @@ func (v *VaultSealWatcher) Run() { default: // Handle the case where the service name doesn't match any of the instances } - v.log.Info("Namespace", v.conf.VaultSecretNameSpace) + podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) @@ -136,13 +136,14 @@ func (v *VaultSealWatcher) Run() { } } else { + v.log.Info("Leader Pod Ip", leaderpodip) leaderaddr, err := vc.LeaderAPIAddr(leaderpodip) if err != nil { v.log.Errorf("failed to retrieve leader address, %s", err) return } v.log.Info("Leader Address", leaderaddr) - podip, err := vc.GetPodIP(svc, "default") + podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) v.log.Info("Unsealing for second % vinstance", podip) if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) From c2a872a629cbc16bd67e79ecabcb9472faaab7e7 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 19 Sep 2023 12:03:22 +0530 Subject: [PATCH 47/64] Modified vault unseal --- internal/job/vault_seal_watcher.go | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 468d00f3..a6436a56 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -32,11 +32,7 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") - // vc, err := client.NewVaultClient(v.log, v.conf) - // if err != nil { - // v.log.Errorf("%s", err) - // return - // } + addresses := []string{ v.conf.Address, v.conf.Address2, @@ -52,19 +48,14 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("Error while retrieving vault instances %s", err) return } - //servicename := []string{"vault-hash-0", "vault-hash-1", "vault-hash-2"} - + var vc *client.VaultClient - var leaderpodip string + var vaultClients []*client.VaultClient for _, address := range addresses { - conf := v.conf // Make a copy of the existing configuration + conf := v.conf conf.Address = address - // conf := config.VaultEnv{ - // Address: address, - // // ReadTimeout: 30, - // // MaxRetries: 3, - // } + v.log.Debug("Address Configuration", conf) vc, err := client.NewVaultClient(v.log, conf) @@ -83,6 +74,7 @@ func (v *VaultSealWatcher) Run() { v.log.Infof("HA ENABLED", v.conf.HAEnabled) for _, svc := range podname { + var leaderpodip string switch svc { case "vault-hash-0": vc = vaultClients[0] @@ -103,13 +95,13 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to retrieve pod ip, %s", err) return } - v.log.Info("POD IP", podip) + res, err := vc.IsVaultSealedForAllInstances(podip) if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) return } - v.log.Info("Seal Status", res) + v.log.Info("Seal Status for %v", podip, res) if res { v.log.Info("vault is sealed, trying to unseal") if svc == "vault-hash-0" { From 719f8b564a02b2234f6970b2e4b9dd4c1923064d Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 19 Sep 2023 12:30:54 +0530 Subject: [PATCH 48/64] Modified vault unseal --- internal/client/vault_seal.go | 9 ++---- internal/job/vault_seal_watcher.go | 48 +++++------------------------- 2 files changed, 10 insertions(+), 47 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 3a114187..cba4b038 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -21,7 +21,6 @@ func (vc *VaultClient) IsVaultSealed() (bool, error) { func (vc *VaultClient) Unseal() error { - vc.log.Info("Unsealing for first instance inside unseal func") status, err := vc.c.Sys().SealStatus() if err != nil { return err @@ -30,14 +29,13 @@ func (vc *VaultClient) Unseal() error { if !status.Sealed { return nil } - vc.log.Info("Status",status) + rootToken, unsealKeys, err := vc.getVaultSecretValues() if err != nil { return err } - vc.log.Info("Root Token",rootToken) - vc.log.Info("Unseal Keys",unsealKeys) + if !status.Initialized && len(rootToken) == 0 && len(unsealKeys) == 0 { vc.log.Debug("intializing vault secret") err = vc.initializeVaultSecret() @@ -59,8 +57,7 @@ func (vc *VaultClient) Unseal() error { func (vc *VaultClient) initializeVaultSecret() error { unsealKeys, rootToken, err := vc.generateUnsealKeys() - vc.log.Info("Unseal Keys",unsealKeys) - vc.log.Info("Root token",rootToken) + if err != nil { return errors.WithMessage(err, "error while generating unseal keys") } diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index a6436a56..5abe7679 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -32,7 +32,7 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") - + var leaderpodip string addresses := []string{ v.conf.Address, v.conf.Address2, @@ -48,16 +48,13 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("Error while retrieving vault instances %s", err) return } - + var vc *client.VaultClient var vaultClients []*client.VaultClient for _, address := range addresses { - conf := v.conf + conf := v.conf conf.Address = address - - v.log.Debug("Address Configuration", conf) - vc, err := client.NewVaultClient(v.log, conf) if err != nil { @@ -67,14 +64,13 @@ func (v *VaultSealWatcher) Run() { vaultClients = append(vaultClients, vc) } - v.log.Debug("Vault Clients", vaultClients) if v.conf.HAEnabled { v.log.Infof("HA ENABLED", v.conf.HAEnabled) for _, svc := range podname { - var leaderpodip string + switch svc { case "vault-hash-0": vc = vaultClients[0] @@ -87,7 +83,7 @@ func (v *VaultSealWatcher) Run() { vc = vaultClients[2] v.log.Debug("Vault Client", vc) default: - // Handle the case where the service name doesn't match any of the instances + } podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) @@ -107,20 +103,13 @@ func (v *VaultSealWatcher) Run() { if svc == "vault-hash-0" { v.log.Info("Unsealing for first instance") - podip, err := vc.GetPodIP(svc, "default") + podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) leaderpodip = podip v.log.Info("Leader Ip", leaderpodip) if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return } - - // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - // if err != nil { - // v.log.Errorf("Failed to fetch the credential: %v\n", err) - // return - // } - //err = vc.UnsealVaultInstance(podip,unsealKeys) err = vc.Unseal() if err != nil { v.log.Errorf("failed to unseal vault, %s", err) @@ -136,7 +125,7 @@ func (v *VaultSealWatcher) Run() { } v.log.Info("Leader Address", leaderaddr) podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) - v.log.Info("Unsealing for second % vinstance", podip) + v.log.Infof("Unsealing for second %v instance", podip) if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return @@ -153,35 +142,12 @@ func (v *VaultSealWatcher) Run() { v.log.Errorf("failed to unseal vault, %s", err) return } - // _, unsealKeys, err := vc.GetVaultSecretValuesforMultiInstance() - // v.log.Debug("Unseal Keys", unsealKeys) - // if err != nil { - // v.log.Errorf("Failed to fetch the credential: %v\n", err) - // return - // } - - // err = vc.UnsealVaultInstance(podip, unsealKeys) - - // if err != nil { - // v.log.Errorf("failed to unseal vault, %s", err) - // return - // } } } } - for _, svc := range podname { - podip, _ := vc.GetPodIP(svc, "default") - res, err := vc.IsVaultSealedForAllInstances(podip) - v.log.Debug("Seal Status of %v :%v", svc, res) - if err != nil { - v.log.Errorf("failed to get vault seal status, %s", err) - return - } - v.log.Infof("vault sealed status: %v", res) - } } } From 5f30d22ca4df65f3027fee39881990a2e2cbf4cc Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 19 Sep 2023 15:12:56 +0530 Subject: [PATCH 49/64] Modified vault unseal --- config/config.go | 1 + internal/job/vault_seal_watcher.go | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/config/config.go b/config/config.go index 6c82efa9..ed711c6e 100644 --- a/config/config.go +++ b/config/config.go @@ -29,6 +29,7 @@ type VaultEnv struct { VaultSecretUnSealKeyPrefix string `envconfig:"VAULT_SECRET_UNSEAL_KEY_PREFIX" default:"unsealkey"` VaultToken string `envconfig:"VAULT_TOKEN"` VaultCredSyncSecretName string `envconfig:"VAULT_CRED_SYNC_SECRET_NAME" default:"vault-cred-sync-data"` + LeaderPodIp string } func FetchConfiguration() (Configuration, error) { diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 5abe7679..b5505322 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -32,7 +32,6 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") - var leaderpodip string addresses := []string{ v.conf.Address, v.conf.Address2, @@ -74,14 +73,20 @@ func (v *VaultSealWatcher) Run() { switch svc { case "vault-hash-0": vc = vaultClients[0] - v.log.Debug("Vault Client", vc) + podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) + if err != nil { + v.log.Errorf("failed to retrieve pod ip, %s", err) + return + } + + v.conf.LeaderPodIp = podip case "vault-hash-1": vc = vaultClients[1] - v.log.Debug("Vault Client", vc) + case "vault-hash-2": vc = vaultClients[2] - v.log.Debug("Vault Client", vc) + default: } @@ -104,8 +109,8 @@ func (v *VaultSealWatcher) Run() { v.log.Info("Unsealing for first instance") podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) - leaderpodip = podip - v.log.Info("Leader Ip", leaderpodip) + v.conf.LeaderPodIp = podip + v.log.Info("Leader Ip", v.conf.LeaderPodIp) if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return @@ -117,20 +122,20 @@ func (v *VaultSealWatcher) Run() { } } else { - v.log.Info("Leader Pod Ip", leaderpodip) - leaderaddr, err := vc.LeaderAPIAddr(leaderpodip) + v.log.Info("Leader Pod Ip", v.conf.LeaderPodIp) + leaderaddr, err := vc.LeaderAPIAddr(v.conf.LeaderPodIp) if err != nil { v.log.Errorf("failed to retrieve leader address, %s", err) return } v.log.Info("Leader Address", leaderaddr) podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) - v.log.Infof("Unsealing for second %v instance", podip) + v.log.Infof("Unsealing for %v instance", podip) if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return } - v.log.Debug("POD IP", podip) + err = vc.JoinRaftCluster(podip, leaderaddr) if err != nil { v.log.Errorf("Failed to join the HA cluster: %v\n", err) From 6bd617e30328bf0fdb6627dfb5432a9f180b66f5 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 19 Sep 2023 15:47:11 +0530 Subject: [PATCH 50/64] Modified vault unseal --- charts/vault-cred/Chart.yaml | 2 +- charts/vault-cred/templates/deployment.yaml | 4 + charts/vault-cred/values.yaml | 4 +- internal/client/vault.go | 24 +--- internal/client/vault_seal.go | 76 +------------ internal/job/vault_seal_watcher.go | 117 +++++++++++--------- server/server.go | 46 ++++---- 7 files changed, 101 insertions(+), 172 deletions(-) diff --git a/charts/vault-cred/Chart.yaml b/charts/vault-cred/Chart.yaml index f9b4a8ae..780774e3 100644 --- a/charts/vault-cred/Chart.yaml +++ b/charts/vault-cred/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.2 +version: 0.1.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/vault-cred/templates/deployment.yaml b/charts/vault-cred/templates/deployment.yaml index 6ffac606..4bcbedf7 100644 --- a/charts/vault-cred/templates/deployment.yaml +++ b/charts/vault-cred/templates/deployment.yaml @@ -43,6 +43,10 @@ spec: value: "{{ .Values.env.logLevel }}" - name: VAULT_ADDR value: "{{ .Values.vault.vaultAddress }}" + - name: VAULT_ADDR2 + value: "{{ .Values.vault.vaultAddress2 }}" + - name: VAULT_ADDR3 + value: "{{ .Values.vault.vaultAddress3 }}" - name: HA_ENABLED value: "{{ .Values.vault.haenabled }}" - name: VAULT_SECRET_NAME diff --git a/charts/vault-cred/values.yaml b/charts/vault-cred/values.yaml index da8183b2..c722b677 100644 --- a/charts/vault-cred/values.yaml +++ b/charts/vault-cred/values.yaml @@ -45,7 +45,9 @@ env: vault: haenabled: true - vaultAddress: http://vault-hash:8200 + vaultAddress: http://vault-hash-1:8200 + vaultAddress2: http://vault-hash-2:8200 + vaultAddress3: http://vault-hash-3:8200 secretName: vault-server secretTokenKeyName: roottoken secretUnSealKeyPrefix: unsealkey diff --git a/internal/client/vault.go b/internal/client/vault.go index bf360151..d925f770 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -179,7 +179,7 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa } func (vc *VaultClient) JoinRaftCluster(podip string, leaderaddress string) error { - // Construct the Vault API address + address := fmt.Sprintf("http://%s:8200", podip) // Set the Vault client address @@ -190,29 +190,13 @@ func (vc *VaultClient) JoinRaftCluster(podip string, leaderaddress string) error vc.log.Debugf("Address: %s", address) - // Extract the leader address from the response - - // Retrieve leader information - // leaderInfo, err := vc.c.Sys().Leader() - // if err != nil { - // return fmt.Errorf("failed to retrieve leader information: %v", err) - // } - - //vc.log.Debugf("Leader address: %s", leaderInfo.LeaderAddress) - - // if leaderInfo.LeaderAddress == "" { - // // Handle the case where leader address is empty - // vc.log.Debug("Leader address is empty") - // return fmt.Errorf("leader address is empty") - // } - req := &api.RaftJoinRequest{ Retry: true, LeaderAPIAddr: leaderaddress, - //LeaderAPIAddr: leaderInfo.LeaderAddress, + } - //vc.log.Debugf("Leader API address: %s", leaderInfo.LeaderAddress) + _, err := vc.c.Sys().RaftJoin(req) if err != nil { @@ -230,7 +214,7 @@ func (vc *VaultClient) LeaderAPIAddr(podip string) (string, error) { return "", err } - vc.log.Debugf("Address: %s", address) + leaderInfo, err := vc.c.Sys().Leader() if err != nil { diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index cba4b038..1da607b5 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -128,87 +128,14 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { return rootToken, unsealKeys, nil } -func (vc *VaultClient) UnsealVaultInstance(podip string, unsealKey []string) error { - // Create a Vault API client - vc.log.Debug("Checking Unseal status for vault Instance") - address := fmt.Sprintf("http://%s:8200", podip) - err := vc.c.SetAddress(address) - if err != nil { - vc.log.Errorf("Error while setting address") - } - vc.log.Debug("Address", address) - - for _, key := range unsealKey { - unsealResponse, err := vc.c.Sys().Unseal(key) - if err != nil { - return errors.WithMessage(err, "error while unsealing") - } - if unsealResponse.Sealed { - vc.log.Debug("Vault is still sealed after unsealing attempt") - } - } - // Check if Vault is sealed and unseal if necessary - // Vault is sealed; unseal it - // unsealResponse, err := vc.c.Sys().Unseal(unsealKey) - // if err != nil { - // return err - // } - // if unsealResponse.Sealed { - // vc.log.Debug("Vault is still sealed after unsealing attempt") - // } - - return nil -} - -func (vc *VaultClient) GetVaultSecretValuesforMultiInstance() (string, []string, error) { - k8s, err := NewK8SClient(vc.log) - if err != nil { - return "", nil, errors.WithMessage(err, "error initializing k8s client") - } - - vaultSec, err := k8s.GetSecret(context.Background(), vc.conf.VaultSecretName, vc.conf.VaultSecretNameSpace) - if err != nil { - if strings.Contains(err.Error(), "secret not found") { - vc.log.Debugf("secret %d not found", vc.conf.VaultSecretName) - return "", nil, nil - } - - return "", nil, errors.WithMessage(err, "error fetching vault secret") - } - - vc.log.Debugf("found %d vault secret values", len(vaultSec.Data)) - unsealKeys := []string{} - var rootToken string - for key, val := range vaultSec.Data { - if strings.HasPrefix(key, vc.conf.VaultSecretUnSealKeyPrefix) { - // decodedValue, err := base64.StdEncoding.DecodeString(val) - if err != nil { - return "", nil, errors.WithMessage(err, "error decoding value") - } - - unsealKeys = append(unsealKeys, val) - vc.log.Debug("Unseal Keys", unsealKeys) - continue - } - if strings.EqualFold(key, vc.conf.VaultSecretTokenKeyName) { - // decodedValue, err := base64.StdEncoding.DecodeString(val) - if err != nil { - return "", nil, errors.WithMessage(err, "error decoding root token") - } - rootToken = val - vc.log.Debug("Root Token Key", rootToken) - } - } - return rootToken, unsealKeys, nil -} func (vc *VaultClient) IsVaultSealedForAllInstances(svc string) (bool, error) { address := fmt.Sprintf("http://%s:8200", svc) err := vc.c.SetAddress(address) - vc.log.Debug("Address for checking vault status", address) + if err != nil { vc.log.Errorf("Error while setting address") } @@ -225,7 +152,6 @@ func (vc *VaultClient) GetPodIP(podName, namespace string) (string, error) { return "", errors.WithMessage(err, "error initializing k8s client") } - // Get the pod's IP address pod, err := k8s.client.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) if err != nil { return "", err diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index b5505322..436f64b1 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -32,64 +32,45 @@ func (v *VaultSealWatcher) CronSpec() string { func (v *VaultSealWatcher) Run() { v.log.Debug("started vault seal watcher job") - addresses := []string{ - v.conf.Address, - v.conf.Address2, - v.conf.Adddress3, - } - k8sclient, err := client.NewK8SClient(v.log) - if err != nil { - v.log.Errorf("Error while connecting with k8s %s", err) - return - } - podname, err := k8sclient.GetVaultPodInstances(context.Background()) - if err != nil { - v.log.Errorf("Error while retrieving vault instances %s", err) - return - } - - var vc *client.VaultClient - var vaultClients []*client.VaultClient - for _, address := range addresses { - conf := v.conf - conf.Address = address - vc, err := client.NewVaultClient(v.log, conf) + if v.conf.HAEnabled { + v.log.Infof(" Vault HA ENABLED", v.conf.HAEnabled) + addresses := []string{ + v.conf.Address, + v.conf.Address2, + v.conf.Adddress3, + } + k8sclient, err := client.NewK8SClient(v.log) if err != nil { - v.log.Errorf("%s", err) + v.log.Errorf("Error while connecting with k8s %s", err) + return + } + podname, err := k8sclient.GetVaultPodInstances(context.Background()) + if err != nil { + v.log.Errorf("Error while retrieving vault instances %s", err) return } - vaultClients = append(vaultClients, vc) - } - - if v.conf.HAEnabled { - - v.log.Infof("HA ENABLED", v.conf.HAEnabled) - - for _, svc := range podname { + var vc *client.VaultClient - switch svc { - case "vault-hash-0": - vc = vaultClients[0] + var vaultClients []*client.VaultClient + for _, address := range addresses { + conf := v.conf + conf.Address = address + vc, err := client.NewVaultClient(v.log, conf) - podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) - if err != nil { - v.log.Errorf("failed to retrieve pod ip, %s", err) - return - } - - v.conf.LeaderPodIp = podip - case "vault-hash-1": - vc = vaultClients[1] + if err != nil { + v.log.Errorf("%s", err) + return + } - case "vault-hash-2": - vc = vaultClients[2] + vaultClients = append(vaultClients, vc) + } - default: + for i, svc := range podname { - } + vc = vaultClients[i] podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) if err != nil { @@ -104,13 +85,13 @@ func (v *VaultSealWatcher) Run() { } v.log.Info("Seal Status for %v", podip, res) if res { - v.log.Info("vault is sealed, trying to unseal") - if svc == "vault-hash-0" { + + if i == 0 { v.log.Info("Unsealing for first instance") podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) v.conf.LeaderPodIp = podip - v.log.Info("Leader Ip", v.conf.LeaderPodIp) + if err != nil { v.log.Errorf("failed to retrieve pod ip, %s", err) return @@ -122,13 +103,13 @@ func (v *VaultSealWatcher) Run() { } } else { - v.log.Info("Leader Pod Ip", v.conf.LeaderPodIp) + leaderaddr, err := vc.LeaderAPIAddr(v.conf.LeaderPodIp) if err != nil { v.log.Errorf("failed to retrieve leader address, %s", err) return } - v.log.Info("Leader Address", leaderaddr) + v.log.Debug("Leader Address", leaderaddr) podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) v.log.Infof("Unsealing for %v instance", podip) if err != nil { @@ -154,5 +135,37 @@ func (v *VaultSealWatcher) Run() { } + } else { + vc, err := client.NewVaultClient(v.log, v.conf) + if err != nil { + v.log.Errorf("%s", err) + return + } + + res, err := vc.IsVaultSealed() + if err != nil { + v.log.Errorf("failed to get vault seal status, %s", err) + return + } + + if res { + v.log.Info("vault is sealed, trying to unseal") + err := vc.Unseal() + if err != nil { + v.log.Errorf("failed to unseal vault, %s", err) + return + } + v.log.Info("vault unsealed executed") + + res, err := vc.IsVaultSealed() + if err != nil { + v.log.Errorf("failed to get vault seal status, %s", err) + return + } + v.log.Infof("vault sealed status: %v", res) + return + } else { + v.log.Debug("vault is in unsealed status") + } } } diff --git a/server/server.go b/server/server.go index a1c065e1..685f4cec 100644 --- a/server/server.go +++ b/server/server.go @@ -75,28 +75,28 @@ func initScheduler(log logging.Logger, cfg config.Configuration) (s *job.Schedul } } - // if cfg.VaultPolicyWatchInterval != "" { - // pj, err := job.NewVaultPolicyWatcher(log, cfg.VaultPolicyWatchInterval) - // if err != nil { - // log.Fatal("failed to init policy watcher job", err) - // } - - // err = s.AddJob("vault-policy-watcher", pj) - // if err != nil { - // log.Fatal("failed to add policy watcher job", err) - // } - // } - - // if cfg.VaultCredSyncInterval != "" { - // pj, err := job.NewVaultCredSync(log, cfg.VaultCredSyncInterval) - // if err != nil { - // log.Fatal("failed to init cred sync job", err) - // } - - // err = s.AddJob("vault-cred-sync", pj) - // if err != nil { - // log.Fatal("failed to add cred sync job", err) - // } - // } + if cfg.VaultPolicyWatchInterval != "" { + pj, err := job.NewVaultPolicyWatcher(log, cfg.VaultPolicyWatchInterval) + if err != nil { + log.Fatal("failed to init policy watcher job", err) + } + + err = s.AddJob("vault-policy-watcher", pj) + if err != nil { + log.Fatal("failed to add policy watcher job", err) + } + } + + if cfg.VaultCredSyncInterval != "" { + pj, err := job.NewVaultCredSync(log, cfg.VaultCredSyncInterval) + if err != nil { + log.Fatal("failed to init cred sync job", err) + } + + err = s.AddJob("vault-cred-sync", pj) + if err != nil { + log.Fatal("failed to add cred sync job", err) + } + } return } From fbc6fcd249e8fbca696e4bd7f99d3c75ad879d7b Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Wed, 20 Sep 2023 12:36:57 +0530 Subject: [PATCH 51/64] Modified HA Enabled --- charts/vault-cred/templates/deployment.yaml | 10 +- charts/vault-cred/values.yaml | 7 +- internal/client/vault_seal.go | 9 +- internal/job/vault_seal_watcher.go | 200 ++++++++++---------- 4 files changed, 116 insertions(+), 110 deletions(-) diff --git a/charts/vault-cred/templates/deployment.yaml b/charts/vault-cred/templates/deployment.yaml index 4bcbedf7..e44beeb4 100644 --- a/charts/vault-cred/templates/deployment.yaml +++ b/charts/vault-cred/templates/deployment.yaml @@ -43,10 +43,12 @@ spec: value: "{{ .Values.env.logLevel }}" - name: VAULT_ADDR value: "{{ .Values.vault.vaultAddress }}" - - name: VAULT_ADDR2 - value: "{{ .Values.vault.vaultAddress2 }}" - - name: VAULT_ADDR3 - value: "{{ .Values.vault.vaultAddress3 }}" + - name: VAULT_SERVER_ADDR1 + value: "{{ .Values.vault.vault_server_address1 }}" + - name: VAULT_SERVER_ADDR2 + value: "{{ .Values.vault.vault_server_address2 }}" + - name: VAULT_SERVER_ADDR3 + value: "{{ .Values.vault.vault_server_address3 }}" - name: HA_ENABLED value: "{{ .Values.vault.haenabled }}" - name: VAULT_SECRET_NAME diff --git a/charts/vault-cred/values.yaml b/charts/vault-cred/values.yaml index c722b677..2df556db 100644 --- a/charts/vault-cred/values.yaml +++ b/charts/vault-cred/values.yaml @@ -45,9 +45,10 @@ env: vault: haenabled: true - vaultAddress: http://vault-hash-1:8200 - vaultAddress2: http://vault-hash-2:8200 - vaultAddress3: http://vault-hash-3:8200 + vaultAddress: http://vault-hash:8200 + vault_server_address1: http://vault-hash-1:8200 + vault_server_address2: http://vault-hash-2:8200 + vault_server_address3: http://vault-hash-3:8200 secretName: vault-server secretTokenKeyName: roottoken secretUnSealKeyPrefix: unsealkey diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 1da607b5..096119d0 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -29,7 +29,6 @@ func (vc *VaultClient) Unseal() error { if !status.Sealed { return nil } - rootToken, unsealKeys, err := vc.getVaultSecretValues() if err != nil { @@ -128,10 +127,6 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { return rootToken, unsealKeys, nil } - - - - func (vc *VaultClient) IsVaultSealedForAllInstances(svc string) (bool, error) { address := fmt.Sprintf("http://%s:8200", svc) err := vc.c.SetAddress(address) @@ -156,6 +151,6 @@ func (vc *VaultClient) GetPodIP(podName, namespace string) (string, error) { if err != nil { return "", err } - vc.log.Debug("Pod ip", pod.Status.PodIP) - return pod.Status.PodIP, nil + vc.log.Debug("Pod Host Name", pod.ObjectMeta.Name) + return pod.ObjectMeta.Name, nil } diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 436f64b1..e35aa82c 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -35,106 +35,12 @@ func (v *VaultSealWatcher) Run() { if v.conf.HAEnabled { v.log.Infof(" Vault HA ENABLED", v.conf.HAEnabled) - - addresses := []string{ - v.conf.Address, - v.conf.Address2, - v.conf.Adddress3, - } - k8sclient, err := client.NewK8SClient(v.log) - if err != nil { - v.log.Errorf("Error while connecting with k8s %s", err) - return - } - podname, err := k8sclient.GetVaultPodInstances(context.Background()) + err := v.Unseal_HA_Enabled() if err != nil { - v.log.Errorf("Error while retrieving vault instances %s", err) + v.log.Errorf("%s", err) return } - var vc *client.VaultClient - - var vaultClients []*client.VaultClient - for _, address := range addresses { - conf := v.conf - conf.Address = address - vc, err := client.NewVaultClient(v.log, conf) - - if err != nil { - v.log.Errorf("%s", err) - return - } - - vaultClients = append(vaultClients, vc) - } - - for i, svc := range podname { - - vc = vaultClients[i] - - podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) - if err != nil { - v.log.Errorf("failed to retrieve pod ip, %s", err) - return - } - - res, err := vc.IsVaultSealedForAllInstances(podip) - if err != nil { - v.log.Errorf("failed to get vault seal status, %s", err) - return - } - v.log.Info("Seal Status for %v", podip, res) - if res { - - if i == 0 { - - v.log.Info("Unsealing for first instance") - podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) - v.conf.LeaderPodIp = podip - - if err != nil { - v.log.Errorf("failed to retrieve pod ip, %s", err) - return - } - err = vc.Unseal() - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return - } - - } else { - - leaderaddr, err := vc.LeaderAPIAddr(v.conf.LeaderPodIp) - if err != nil { - v.log.Errorf("failed to retrieve leader address, %s", err) - return - } - v.log.Debug("Leader Address", leaderaddr) - podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) - v.log.Infof("Unsealing for %v instance", podip) - if err != nil { - v.log.Errorf("failed to retrieve pod ip, %s", err) - return - } - - err = vc.JoinRaftCluster(podip, leaderaddr) - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - return - - } - err = vc.Unseal() - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return - } - - } - - } - - } - } else { vc, err := client.NewVaultClient(v.log, v.conf) if err != nil { @@ -169,3 +75,105 @@ func (v *VaultSealWatcher) Run() { } } } +func (v *VaultSealWatcher) Unseal_HA_Enabled() error { + + addresses := []string{ + v.conf.Address, + v.conf.Address2, + v.conf.Adddress3, + } + k8sclient, err := client.NewK8SClient(v.log) + if err != nil { + v.log.Errorf("Error while connecting with k8s %s", err) + return err + } + podname, err := k8sclient.GetVaultPodInstances(context.Background()) + if err != nil { + v.log.Errorf("Error while retrieving vault instances %s", err) + return err + } + + var vc *client.VaultClient + + var vaultClients []*client.VaultClient + for _, address := range addresses { + conf := v.conf + conf.Address = address + vc, err := client.NewVaultClient(v.log, conf) + + if err != nil { + v.log.Errorf("%s", err) + return err + } + + vaultClients = append(vaultClients, vc) + } + + for i, svc := range podname { + + vc = vaultClients[i] + + // podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) + // if err != nil { + // v.log.Errorf("failed to retrieve pod ip, %s", err) + // return err + // } + + res, err := vc.IsVaultSealedForAllInstances(svc) + if err != nil { + v.log.Errorf("failed to get vault seal status, %s", err) + return err + } + v.log.Info("Seal Status for %v", svc, res) + if res { + + if i == 0 { + + v.log.Info("Unsealing for first instance") + // podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) + v.conf.LeaderPodIp = svc + + if err != nil { + v.log.Errorf("failed to retrieve pod ip, %s", err) + return err + } + err = vc.Unseal() + if err != nil { + v.log.Errorf("failed to unseal vault, %s", err) + return err + } + + } else { + + leaderaddr, err := vc.LeaderAPIAddr(v.conf.LeaderPodIp) + if err != nil { + v.log.Errorf("failed to retrieve leader address, %s", err) + return err + } + v.log.Debug("Leader Address", leaderaddr) + // podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) + v.log.Infof("Unsealing for %v instance", podname) + // if err != nil { + // v.log.Errorf("failed to retrieve pod ip, %s", err) + // return err + // } + + err = vc.JoinRaftCluster(svc, leaderaddr) + if err != nil { + v.log.Errorf("Failed to join the HA cluster: %v\n", err) + return err + + } + err = vc.Unseal() + if err != nil { + v.log.Errorf("failed to unseal vault, %s", err) + return err + } + + } + + } + + } + return nil +} From 21e44829b866348197d48851100c4f427456333c Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Wed, 20 Sep 2023 12:54:44 +0530 Subject: [PATCH 52/64] Modified HA Enabled --- internal/client/vault_seal.go | 15 --------------- internal/job/vault_seal_watcher.go | 16 +++------------- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index 096119d0..a309fb49 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -8,7 +8,6 @@ import ( "github.com/hashicorp/vault/api" "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (vc *VaultClient) IsVaultSealed() (bool, error) { @@ -140,17 +139,3 @@ func (vc *VaultClient) IsVaultSealedForAllInstances(svc string) (bool, error) { } return status.Sealed, nil } - -func (vc *VaultClient) GetPodIP(podName, namespace string) (string, error) { - k8s, err := NewK8SClient(vc.log) - if err != nil { - return "", errors.WithMessage(err, "error initializing k8s client") - } - - pod, err := k8s.client.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) - if err != nil { - return "", err - } - vc.log.Debug("Pod Host Name", pod.ObjectMeta.Name) - return pod.ObjectMeta.Name, nil -} diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index e35aa82c..6109b88f 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -76,7 +76,7 @@ func (v *VaultSealWatcher) Run() { } } func (v *VaultSealWatcher) Unseal_HA_Enabled() error { - + addresses := []string{ v.conf.Address, v.conf.Address2, @@ -113,12 +113,6 @@ func (v *VaultSealWatcher) Unseal_HA_Enabled() error { vc = vaultClients[i] - // podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) - // if err != nil { - // v.log.Errorf("failed to retrieve pod ip, %s", err) - // return err - // } - res, err := vc.IsVaultSealedForAllInstances(svc) if err != nil { v.log.Errorf("failed to get vault seal status, %s", err) @@ -130,7 +124,7 @@ func (v *VaultSealWatcher) Unseal_HA_Enabled() error { if i == 0 { v.log.Info("Unsealing for first instance") - // podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) + v.conf.LeaderPodIp = svc if err != nil { @@ -151,12 +145,8 @@ func (v *VaultSealWatcher) Unseal_HA_Enabled() error { return err } v.log.Debug("Leader Address", leaderaddr) - // podip, err := vc.GetPodIP(svc, v.conf.VaultSecretNameSpace) + v.log.Infof("Unsealing for %v instance", podname) - // if err != nil { - // v.log.Errorf("failed to retrieve pod ip, %s", err) - // return err - // } err = vc.JoinRaftCluster(svc, leaderaddr) if err != nil { From deee900f77a8ddd2d7942711e176710c0a85e30b Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Wed, 20 Sep 2023 12:59:36 +0530 Subject: [PATCH 53/64] Vault unseal handling for vault HA with Raft --- charts/vault-cred/Chart.yaml | 2 +- charts/vault-cred/templates/deployment.yaml | 8 +- charts/vault-cred/values.yaml | 4 +- config/config.go | 8 +- internal/client/k8s.go | 27 +-- internal/client/vault.go | 50 +----- internal/client/vault_seal.go | 14 -- internal/job/vault_seal_watcher.go | 172 +++++++++----------- 8 files changed, 87 insertions(+), 198 deletions(-) diff --git a/charts/vault-cred/Chart.yaml b/charts/vault-cred/Chart.yaml index 780774e3..7e932e5b 100644 --- a/charts/vault-cred/Chart.yaml +++ b/charts/vault-cred/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.3 +version: 0.1.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/vault-cred/templates/deployment.yaml b/charts/vault-cred/templates/deployment.yaml index e44beeb4..835e91d2 100644 --- a/charts/vault-cred/templates/deployment.yaml +++ b/charts/vault-cred/templates/deployment.yaml @@ -43,12 +43,8 @@ spec: value: "{{ .Values.env.logLevel }}" - name: VAULT_ADDR value: "{{ .Values.vault.vaultAddress }}" - - name: VAULT_SERVER_ADDR1 - value: "{{ .Values.vault.vault_server_address1 }}" - - name: VAULT_SERVER_ADDR2 - value: "{{ .Values.vault.vault_server_address2 }}" - - name: VAULT_SERVER_ADDR3 - value: "{{ .Values.vault.vault_server_address3 }}" + - name: VAULT_NODE_ADDRESSES + value: "{{ .Values.vault.vaultNodeAddresses }}" - name: HA_ENABLED value: "{{ .Values.vault.haenabled }}" - name: VAULT_SECRET_NAME diff --git a/charts/vault-cred/values.yaml b/charts/vault-cred/values.yaml index 2df556db..05023e94 100644 --- a/charts/vault-cred/values.yaml +++ b/charts/vault-cred/values.yaml @@ -46,9 +46,7 @@ env: vault: haenabled: true vaultAddress: http://vault-hash:8200 - vault_server_address1: http://vault-hash-1:8200 - vault_server_address2: http://vault-hash-2:8200 - vault_server_address3: http://vault-hash-3:8200 + vaultNodeAddresses: "http://vault-hash-0:8200,http://vault-hash-1:8200,http://vault-hash-2:8200" secretName: vault-server secretTokenKeyName: roottoken secretUnSealKeyPrefix: unsealkey diff --git a/config/config.go b/config/config.go index ed711c6e..81edd0d5 100644 --- a/config/config.go +++ b/config/config.go @@ -16,20 +16,18 @@ type Configuration struct { type VaultEnv struct { HAEnabled bool `envconfig:"HA_ENABLED" default:"true"` - Address string `envconfig:"VAULT_ADDR" default:"http://vault-hash-0:8200"` - Address2 string `envconfig:"VAULT_ADDR2" default:"http://vault-hash-1:8200"` - Adddress3 string `envconfig:"VAULT_ADDR3" default:"http://vault-hash-2:8200"` + Address string `envconfig:"VAULT_ADDR" required:"true"` + NodeAddresses []string `envconfig:"VAULT_NODE_ADDRESSES" required:"true"` CACert string `envconfig:"VAULT_CACERT" required:"false"` ReadTimeout time.Duration `envconfig:"VAULT_READ_TIMEOUT" default:"60s"` MaxRetries int `envconfig:"VAULT_MAX_RETRIES" default:"5"` VaultTokenForRequests bool `envconfig:"VAULT_TOKEN_FOR_REQUESTS" default:"false"` VaultSecretName string `envconfig:"VAULT_SECRET_NAME" default:"vault-server"` - VaultSecretNameSpace string `envconfig:"POD_NAMESPACE" default:"default" required:"true"` + VaultSecretNameSpace string `envconfig:"POD_NAMESPACE" required:"true"` VaultSecretTokenKeyName string `envconfig:"VAULT_SECRET_TOKEN_KEY_NAME" default:"root-token"` VaultSecretUnSealKeyPrefix string `envconfig:"VAULT_SECRET_UNSEAL_KEY_PREFIX" default:"unsealkey"` VaultToken string `envconfig:"VAULT_TOKEN"` VaultCredSyncSecretName string `envconfig:"VAULT_CRED_SYNC_SECRET_NAME" default:"vault-cred-sync-data"` - LeaderPodIp string } func FetchConfiguration() (Configuration, error) { diff --git a/internal/client/k8s.go b/internal/client/k8s.go index 5c8e04a7..a0a0f5be 100644 --- a/internal/client/k8s.go +++ b/internal/client/k8s.go @@ -2,7 +2,6 @@ package client import ( "context" - "sort" "strings" "time" @@ -14,9 +13,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) -const ( - labelSelector = "app.kubernetes.io/name=vault" -) + type K8SClient struct { client *kubernetes.Clientset log logging.Logger @@ -135,25 +132,3 @@ func (k *K8SClient) GetConfigMapsHasPrefix(ctx context.Context, prefix string) ( } return allConfigMapData, nil } - -func (k *K8SClient) GetVaultPodInstances(ctx context.Context) ([]string, error) { - var podnames []string - - - pods, err := k.client.CoreV1().Pods("default").List(context.TODO(), metav1.ListOptions{ - LabelSelector: labelSelector, - }) - if err != nil { - return nil,errors.WithMessage(err, "error while retrieving the pods ") - } - - - for _, pod := range pods.Items { - podnames = append(podnames, pod.Name) - - - } - sort.Strings(podnames) - - return podnames,nil -} diff --git a/internal/client/vault.go b/internal/client/vault.go index d925f770..c3b95cb0 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -73,8 +73,7 @@ func NewVaultClientForVaultToken(log logging.Logger, conf config.VaultEnv) (*Vau func NewVaultClient(log logging.Logger, conf config.VaultEnv) (*VaultClient, error) { cfg, err := prepareVaultConfig(conf) if err != nil { - log.Debug("Error while preparing vault Config") - return nil, err + return nil, fmt.Errorf("error in vault config, %v", err) } c, err := api.NewClient(cfg) @@ -178,58 +177,15 @@ func (vc *VaultClient) DeleteCredential(ctx context.Context, mountPath, secretPa return } -func (vc *VaultClient) JoinRaftCluster(podip string, leaderaddress string) error { - - address := fmt.Sprintf("http://%s:8200", podip) - - // Set the Vault client address - if err := vc.c.SetAddress(address); err != nil { - vc.log.Debug("failed to set Vault client address: %v", err) - return err - } - - vc.log.Debugf("Address: %s", address) - +func (vc *VaultClient) JoinRaftCluster(leaderAddress string) error { req := &api.RaftJoinRequest{ Retry: true, - LeaderAPIAddr: leaderaddress, - + LeaderAPIAddr: leaderAddress, } - - _, err := vc.c.Sys().RaftJoin(req) if err != nil { return fmt.Errorf("failed to join the Raft cluster: %v", err) } - return nil } - -func (vc *VaultClient) LeaderAPIAddr(podip string) (string, error) { - address := fmt.Sprintf("http://%s:8200", podip) - - if err := vc.c.SetAddress(address); err != nil { - vc.log.Debug("failed to set Vault client address: %v", err) - return "", err - } - - - - leaderInfo, err := vc.c.Sys().Leader() - if err != nil { - vc.log.Error("failed to retrieve leader information: %v", err) - return "", err - } - - vc.log.Debugf("Leader address: %s", leaderInfo.LeaderAddress) - - if leaderInfo.LeaderAddress == "" { - - vc.log.Debug("Leader address is empty") - } - leaderaddress := leaderInfo.LeaderAddress - - return leaderaddress, nil - -} diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index a309fb49..f7a4e6f8 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -125,17 +125,3 @@ func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { } return rootToken, unsealKeys, nil } - -func (vc *VaultClient) IsVaultSealedForAllInstances(svc string) (bool, error) { - address := fmt.Sprintf("http://%s:8200", svc) - err := vc.c.SetAddress(address) - - if err != nil { - vc.log.Errorf("Error while setting address") - } - status, err := vc.c.Sys().SealStatus() - if err != nil { - return false, err - } - return status.Sealed, nil -} diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 6109b88f..3d6aaaa0 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -1,7 +1,7 @@ package job import ( - "context" + "fmt" "github.com/intelops/go-common/logging" "github.com/intelops/vault-cred/config" @@ -31,139 +31,119 @@ func (v *VaultSealWatcher) CronSpec() string { } func (v *VaultSealWatcher) Run() { - v.log.Debug("started vault seal watcher job") + v.log.Debugf("started vault seal watcher job with vault HA: %v", v.conf.HAEnabled) if v.conf.HAEnabled { - v.log.Infof(" Vault HA ENABLED", v.conf.HAEnabled) - err := v.Unseal_HA_Enabled() - if err != nil { - v.log.Errorf("%s", err) + if len(v.conf.NodeAddresses) != 3 { + v.log.Errorf("vault HA node count %d is not valid", len(v.conf.NodeAddresses)) return } - } else { - vc, err := client.NewVaultClient(v.log, v.conf) - if err != nil { + if err := v.handleUnsealForHAVault(); err != nil { v.log.Errorf("%s", err) - return - } - - res, err := vc.IsVaultSealed() - if err != nil { - v.log.Errorf("failed to get vault seal status, %s", err) - return } - - if res { - v.log.Info("vault is sealed, trying to unseal") - err := vc.Unseal() - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return - } - v.log.Info("vault unsealed executed") - - res, err := vc.IsVaultSealed() - if err != nil { - v.log.Errorf("failed to get vault seal status, %s", err) - return - } - v.log.Infof("vault sealed status: %v", res) - return - } else { - v.log.Debug("vault is in unsealed status") + } else { + if err := v.handleUnsealForNonHAVault(); err != nil { + v.log.Errorf("%s", err) } } } -func (v *VaultSealWatcher) Unseal_HA_Enabled() error { - addresses := []string{ - v.conf.Address, - v.conf.Address2, - v.conf.Adddress3, - } - k8sclient, err := client.NewK8SClient(v.log) +func (v *VaultSealWatcher) handleUnsealForNonHAVault() error { + vc, err := client.NewVaultClient(v.log, v.conf) if err != nil { - v.log.Errorf("Error while connecting with k8s %s", err) return err } - podname, err := k8sclient.GetVaultPodInstances(context.Background()) + + res, err := vc.IsVaultSealed() if err != nil { - v.log.Errorf("Error while retrieving vault instances %s", err) - return err + return fmt.Errorf("failed to get vault seal status, %s", err) } - var vc *client.VaultClient + if res { + v.log.Info("vault is sealed, trying to unseal") + err := vc.Unseal() + if err != nil { + return fmt.Errorf("failed to unseal vault, %s", err) + } + v.log.Info("vault unsealed executed") + + res, err := vc.IsVaultSealed() + if err != nil { + return fmt.Errorf("failed to get vault seal status, %s", err) + } + v.log.Infof("vault sealed status: %v", res) + } else { + v.log.Debug("vault is in unsealed status") + } + return nil +} +func (v *VaultSealWatcher) handleUnsealForHAVault() error { var vaultClients []*client.VaultClient - for _, address := range addresses { + for _, nodeAddress := range v.conf.NodeAddresses { conf := v.conf - conf.Address = address + conf.Address = nodeAddress vc, err := client.NewVaultClient(v.log, conf) - if err != nil { - v.log.Errorf("%s", err) return err } vaultClients = append(vaultClients, vc) } - for i, svc := range podname { - - vc = vaultClients[i] + allSealed, err := v.isAllNodesSealed(vaultClients) + if err != nil { + return err + } - res, err := vc.IsVaultSealedForAllInstances(svc) + if allSealed { + v.log.Info("vault is sealed for all nodes") + err = vaultClients[0].Unseal() if err != nil { - v.log.Errorf("failed to get vault seal status, %s", err) - return err + return fmt.Errorf("failed to unseal vault for leader node, %v", err) } - v.log.Info("Seal Status for %v", svc, res) - if res { - - if i == 0 { - v.log.Info("Unsealing for first instance") - - v.conf.LeaderPodIp = svc - - if err != nil { - v.log.Errorf("failed to retrieve pod ip, %s", err) - return err - } - err = vc.Unseal() - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return err - } - - } else { - - leaderaddr, err := vc.LeaderAPIAddr(v.conf.LeaderPodIp) - if err != nil { - v.log.Errorf("failed to retrieve leader address, %s", err) - return err - } - v.log.Debug("Leader Address", leaderaddr) + err = vaultClients[1].JoinRaftCluster(v.conf.NodeAddresses[0]) + if err != nil { + return fmt.Errorf("failed to join the HA cluster by 2nd node, %v", err) + } - v.log.Infof("Unsealing for %v instance", podname) + err = vaultClients[1].Unseal() + if err != nil { + return fmt.Errorf("failed to unseal vault for 2nd node, %v", err) + } - err = vc.JoinRaftCluster(svc, leaderaddr) - if err != nil { - v.log.Errorf("Failed to join the HA cluster: %v\n", err) - return err + err = vaultClients[2].JoinRaftCluster(v.conf.NodeAddresses[0]) + if err != nil { + return fmt.Errorf("failed to join the HA cluster by 3rd node, %v", err) + } - } - err = vc.Unseal() - if err != nil { - v.log.Errorf("failed to unseal vault, %s", err) - return err - } + err = vaultClients[2].Unseal() + if err != nil { + return fmt.Errorf("failed to unseal vault for 3rd node, %v", err) + } - } + v.log.Info("vault is unsealed for all nodes") + } else { + v.log.Info("some vault nodes are sealed") + } + return nil +} +func (v *VaultSealWatcher) isAllNodesSealed(vaultClients []*client.VaultClient) (bool, error) { + status := false + for _, vc := range vaultClients { + res, err := vc.IsVaultSealed() + if err != nil { + return false, fmt.Errorf("failed to get vault seal status for %s, %v", v.conf.Address, err) } + if !res { + return false, nil + } + v.log.Info("vault node %s is sealed", v.conf.Address) + status = res } - return nil + return status, nil } From dd1d8e083ac267ee31fcdbe531d43f2a32747c08 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sun, 24 Sep 2023 13:32:46 +0530 Subject: [PATCH 54/64] Refactoring for vaultHA --- internal/client/vault.go | 3 +- internal/job/vault_seal_watcher.go | 90 ++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/internal/client/vault.go b/internal/client/vault.go index c3b95cb0..69eb609a 100644 --- a/internal/client/vault.go +++ b/internal/client/vault.go @@ -183,9 +183,10 @@ func (vc *VaultClient) JoinRaftCluster(leaderAddress string) error { LeaderAPIAddr: leaderAddress, } - _, err := vc.c.Sys().RaftJoin(req) + res, err := vc.c.Sys().RaftJoin(req) if err != nil { return fmt.Errorf("failed to join the Raft cluster: %v", err) } + vc.log.Debug("Raft Joined status", res.Joined) return nil } diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 3d6aaaa0..25999c65 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -81,6 +81,7 @@ func (v *VaultSealWatcher) handleUnsealForNonHAVault() error { func (v *VaultSealWatcher) handleUnsealForHAVault() error { var vaultClients []*client.VaultClient + //var sealed bool for _, nodeAddress := range v.conf.NodeAddresses { conf := v.conf conf.Address = nodeAddress @@ -91,43 +92,70 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { vaultClients = append(vaultClients, vc) } - - allSealed, err := v.isAllNodesSealed(vaultClients) - if err != nil { - return err - } - - if allSealed { - v.log.Info("vault is sealed for all nodes") - err = vaultClients[0].Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault for leader node, %v", err) - } - - err = vaultClients[1].JoinRaftCluster(v.conf.NodeAddresses[0]) - if err != nil { - return fmt.Errorf("failed to join the HA cluster by 2nd node, %v", err) - } - - err = vaultClients[1].Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault for 2nd node, %v", err) - } - - err = vaultClients[2].JoinRaftCluster(v.conf.NodeAddresses[0]) + for _, vc := range vaultClients { + sealed, err := vc.IsVaultSealed() if err != nil { - return fmt.Errorf("failed to join the HA cluster by 3rd node, %v", err) + return err } + if sealed { + switch vc { + case vaultClients[0]: + err = vc.Unseal() + if err != nil { + return fmt.Errorf("failed to unseal vault for leader node, %v", err) + } + default: + err = vc.JoinRaftCluster(v.conf.NodeAddresses[0]) + if err != nil { + return fmt.Errorf("failed to join the HA cluster, %v", err) + } + + err = vc.Unseal() + if err != nil { + return fmt.Errorf("failed to unseal vault, %v", err) + } + } - err = vaultClients[2].Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault for 3rd node, %v", err) } - v.log.Info("vault is unsealed for all nodes") - } else { - v.log.Info("some vault nodes are sealed") } + + // allSealed, err := v.isAllNodesSealed(vaultClients) + // if err != nil { + // return err + // } + + // if sealed { + // v.log.Info("vault is sealed for all nodes") + // err = vaultClients[0].Unseal() + // if err != nil { + // return fmt.Errorf("failed to unseal vault for leader node, %v", err) + // } + + // err = vaultClients[1].JoinRaftCluster(v.conf.NodeAddresses[0]) + // if err != nil { + // return fmt.Errorf("failed to join the HA cluster by 2nd node, %v", err) + // } + + // err = vaultClients[1].Unseal() + // if err != nil { + // return fmt.Errorf("failed to unseal vault for 2nd node, %v", err) + // } + + // err = vaultClients[2].JoinRaftCluster(v.conf.NodeAddresses[0]) + // if err != nil { + // return fmt.Errorf("failed to join the HA cluster by 3rd node, %v", err) + // } + + // err = vaultClients[2].Unseal() + // if err != nil { + // return fmt.Errorf("failed to unseal vault for 3rd node, %v", err) + // } + + // v.log.Info("vault is unsealed for all nodes") + // } else { + // v.log.Info("some vault nodes are sealed") + // } return nil } From a3a9ae1a7ab975e4e3afc579374deb891321bbff Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sun, 24 Sep 2023 13:41:52 +0530 Subject: [PATCH 55/64] Refactoring for vaultHA --- internal/job/vault_seal_watcher.go | 53 ------------------------------ 1 file changed, 53 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 25999c65..9a59ade3 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -120,58 +120,5 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { } - // allSealed, err := v.isAllNodesSealed(vaultClients) - // if err != nil { - // return err - // } - - // if sealed { - // v.log.Info("vault is sealed for all nodes") - // err = vaultClients[0].Unseal() - // if err != nil { - // return fmt.Errorf("failed to unseal vault for leader node, %v", err) - // } - - // err = vaultClients[1].JoinRaftCluster(v.conf.NodeAddresses[0]) - // if err != nil { - // return fmt.Errorf("failed to join the HA cluster by 2nd node, %v", err) - // } - - // err = vaultClients[1].Unseal() - // if err != nil { - // return fmt.Errorf("failed to unseal vault for 2nd node, %v", err) - // } - - // err = vaultClients[2].JoinRaftCluster(v.conf.NodeAddresses[0]) - // if err != nil { - // return fmt.Errorf("failed to join the HA cluster by 3rd node, %v", err) - // } - - // err = vaultClients[2].Unseal() - // if err != nil { - // return fmt.Errorf("failed to unseal vault for 3rd node, %v", err) - // } - - // v.log.Info("vault is unsealed for all nodes") - // } else { - // v.log.Info("some vault nodes are sealed") - // } return nil } - -func (v *VaultSealWatcher) isAllNodesSealed(vaultClients []*client.VaultClient) (bool, error) { - status := false - for _, vc := range vaultClients { - res, err := vc.IsVaultSealed() - if err != nil { - return false, fmt.Errorf("failed to get vault seal status for %s, %v", v.conf.Address, err) - } - - if !res { - return false, nil - } - v.log.Info("vault node %s is sealed", v.conf.Address) - status = res - } - return status, nil -} From 8407660f670499c8c960b24633e74066f82c42c0 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Mon, 25 Sep 2023 11:26:06 +0530 Subject: [PATCH 56/64] Testing the commit --- internal/job/vault_seal_watcher.go | 71 ++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 9a59ade3..3d6aaaa0 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -81,7 +81,6 @@ func (v *VaultSealWatcher) handleUnsealForNonHAVault() error { func (v *VaultSealWatcher) handleUnsealForHAVault() error { var vaultClients []*client.VaultClient - //var sealed bool for _, nodeAddress := range v.conf.NodeAddresses { conf := v.conf conf.Address = nodeAddress @@ -92,33 +91,59 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { vaultClients = append(vaultClients, vc) } - for _, vc := range vaultClients { - sealed, err := vc.IsVaultSealed() + + allSealed, err := v.isAllNodesSealed(vaultClients) + if err != nil { + return err + } + + if allSealed { + v.log.Info("vault is sealed for all nodes") + err = vaultClients[0].Unseal() if err != nil { - return err + return fmt.Errorf("failed to unseal vault for leader node, %v", err) } - if sealed { - switch vc { - case vaultClients[0]: - err = vc.Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault for leader node, %v", err) - } - default: - err = vc.JoinRaftCluster(v.conf.NodeAddresses[0]) - if err != nil { - return fmt.Errorf("failed to join the HA cluster, %v", err) - } - - err = vc.Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault, %v", err) - } - } + err = vaultClients[1].JoinRaftCluster(v.conf.NodeAddresses[0]) + if err != nil { + return fmt.Errorf("failed to join the HA cluster by 2nd node, %v", err) } - } + err = vaultClients[1].Unseal() + if err != nil { + return fmt.Errorf("failed to unseal vault for 2nd node, %v", err) + } + + err = vaultClients[2].JoinRaftCluster(v.conf.NodeAddresses[0]) + if err != nil { + return fmt.Errorf("failed to join the HA cluster by 3rd node, %v", err) + } + err = vaultClients[2].Unseal() + if err != nil { + return fmt.Errorf("failed to unseal vault for 3rd node, %v", err) + } + + v.log.Info("vault is unsealed for all nodes") + } else { + v.log.Info("some vault nodes are sealed") + } return nil } + +func (v *VaultSealWatcher) isAllNodesSealed(vaultClients []*client.VaultClient) (bool, error) { + status := false + for _, vc := range vaultClients { + res, err := vc.IsVaultSealed() + if err != nil { + return false, fmt.Errorf("failed to get vault seal status for %s, %v", v.conf.Address, err) + } + + if !res { + return false, nil + } + v.log.Info("vault node %s is sealed", v.conf.Address) + status = res + } + return status, nil +} From 26629a1fd76703e45bfd4cee7a2a2794a4ba445a Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 26 Sep 2023 16:58:09 +0530 Subject: [PATCH 57/64] Added LeaderApiAddress --- charts/vault-cred/Chart.yaml | 2 +- charts/vault-cred/templates/deployment.yaml | 2 ++ charts/vault-cred/values.yaml | 1 + config/config.go | 1 + internal/job/vault_seal_watcher.go | 11 ++++++++--- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/charts/vault-cred/Chart.yaml b/charts/vault-cred/Chart.yaml index 7e932e5b..ee9d4419 100644 --- a/charts/vault-cred/Chart.yaml +++ b/charts/vault-cred/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.4 +version: 0.1.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/vault-cred/templates/deployment.yaml b/charts/vault-cred/templates/deployment.yaml index 835e91d2..58fcce8a 100644 --- a/charts/vault-cred/templates/deployment.yaml +++ b/charts/vault-cred/templates/deployment.yaml @@ -43,6 +43,8 @@ spec: value: "{{ .Values.env.logLevel }}" - name: VAULT_ADDR value: "{{ .Values.vault.vaultAddress }}" + - name: VAULT_LEADER_ADDRESS + value: "{{ .Values.vault.vaultLeaderAddress }}" - name: VAULT_NODE_ADDRESSES value: "{{ .Values.vault.vaultNodeAddresses }}" - name: HA_ENABLED diff --git a/charts/vault-cred/values.yaml b/charts/vault-cred/values.yaml index 05023e94..a2cd113c 100644 --- a/charts/vault-cred/values.yaml +++ b/charts/vault-cred/values.yaml @@ -46,6 +46,7 @@ env: vault: haenabled: true vaultAddress: http://vault-hash:8200 + vaultLeaderAddress: vault-hash-0.vault-hash-internal:8200 vaultNodeAddresses: "http://vault-hash-0:8200,http://vault-hash-1:8200,http://vault-hash-2:8200" secretName: vault-server secretTokenKeyName: roottoken diff --git a/config/config.go b/config/config.go index 81edd0d5..372e3ab6 100644 --- a/config/config.go +++ b/config/config.go @@ -18,6 +18,7 @@ type VaultEnv struct { HAEnabled bool `envconfig:"HA_ENABLED" default:"true"` Address string `envconfig:"VAULT_ADDR" required:"true"` NodeAddresses []string `envconfig:"VAULT_NODE_ADDRESSES" required:"true"` + LeaderAPIAddr string `envconfig:"VAULT_LEADER_ADDRESS" required:"true"` CACert string `envconfig:"VAULT_CACERT" required:"false"` ReadTimeout time.Duration `envconfig:"VAULT_READ_TIMEOUT" default:"60s"` MaxRetries int `envconfig:"VAULT_MAX_RETRIES" default:"5"` diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 3d6aaaa0..242a30d6 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -104,7 +104,7 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { return fmt.Errorf("failed to unseal vault for leader node, %v", err) } - err = vaultClients[1].JoinRaftCluster(v.conf.NodeAddresses[0]) + err = vaultClients[1].JoinRaftCluster(v.conf.LeaderAPIAddr) if err != nil { return fmt.Errorf("failed to join the HA cluster by 2nd node, %v", err) } @@ -114,7 +114,7 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { return fmt.Errorf("failed to unseal vault for 2nd node, %v", err) } - err = vaultClients[2].JoinRaftCluster(v.conf.NodeAddresses[0]) + err = vaultClients[2].JoinRaftCluster(v.conf.LeaderAPIAddr) if err != nil { return fmt.Errorf("failed to join the HA cluster by 3rd node, %v", err) } @@ -126,7 +126,9 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { v.log.Info("vault is unsealed for all nodes") } else { + v.log.Info("some vault nodes are sealed") + } return nil } @@ -135,14 +137,17 @@ func (v *VaultSealWatcher) isAllNodesSealed(vaultClients []*client.VaultClient) status := false for _, vc := range vaultClients { res, err := vc.IsVaultSealed() + v.log.Info("unseal status for %v is %v", vc, res) if err != nil { return false, fmt.Errorf("failed to get vault seal status for %s, %v", v.conf.Address, err) } if !res { return false, nil + } else { + v.log.Info("vault node %s is sealed", v.conf.Address) } - v.log.Info("vault node %s is sealed", v.conf.Address) + status = res } return status, nil From fd118ec028fc6724656944297394e5631b8e02c8 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 26 Sep 2023 17:30:55 +0530 Subject: [PATCH 58/64] Modified Seal Watcher --- internal/job/vault_seal_watcher.go | 72 ++++++++++++++---------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 242a30d6..9f03503a 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -80,60 +80,57 @@ func (v *VaultSealWatcher) handleUnsealForNonHAVault() error { } func (v *VaultSealWatcher) handleUnsealForHAVault() error { - var vaultClients []*client.VaultClient - for _, nodeAddress := range v.conf.NodeAddresses { + var followerClients []*client.VaultClient + var leaderVaultClients []*client.VaultClient + for index, nodeAddress := range v.conf.NodeAddresses { conf := v.conf conf.Address = nodeAddress vc, err := client.NewVaultClient(v.log, conf) if err != nil { return err } - - vaultClients = append(vaultClients, vc) + // assuming that first node address is the leader + // Todo: keep leaders and followers in the configuration separate + if index == 0 { + leaderVaultClients = append(leaderVaultClients, vc) + } else { + followerClients = append(followerClients, vc) + } } - allSealed, err := v.isAllNodesSealed(vaultClients) - if err != nil { + if leaderSealed, err := v.areNodesSealed(leaderVaultClients); err != nil { return err - } - - if allSealed { - v.log.Info("vault is sealed for all nodes") - err = vaultClients[0].Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault for leader node, %v", err) - } - - err = vaultClients[1].JoinRaftCluster(v.conf.LeaderAPIAddr) - if err != nil { - return fmt.Errorf("failed to join the HA cluster by 2nd node, %v", err) - } - - err = vaultClients[1].Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault for 2nd node, %v", err) - } - - err = vaultClients[2].JoinRaftCluster(v.conf.LeaderAPIAddr) - if err != nil { - return fmt.Errorf("failed to join the HA cluster by 3rd node, %v", err) + } else if leaderSealed { + v.log.Info("vault is sealed for leader nodes") + for _, vc := range leaderVaultClients { + err = vc.Unseal() + if err != nil { + return fmt.Errorf("failed to unseal vault for leader node, %v", err) + } } + } - err = vaultClients[2].Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault for 3rd node, %v", err) + if followersSealed, err := v.areNodesSealed(followerClients); err != nil { + return err + } else if followersSealed { + for index, vc := range followerClients { + err = vc.JoinRaftCluster(v.conf.LeaderAPIAddr) + if err != nil { + return fmt.Errorf("failed to join the HA cluster by node index: %v, %v", index+1, err) + } + + err = vc.Unseal() + if err != nil { + return fmt.Errorf("failed to unseal vault for node index: %v, %v", index+1, err) + } } - v.log.Info("vault is unsealed for all nodes") - } else { - - v.log.Info("some vault nodes are sealed") - } + return nil } -func (v *VaultSealWatcher) isAllNodesSealed(vaultClients []*client.VaultClient) (bool, error) { +func (v *VaultSealWatcher) areNodesSealed(vaultClients []*client.VaultClient) (bool, error) { status := false for _, vc := range vaultClients { res, err := vc.IsVaultSealed() @@ -147,7 +144,6 @@ func (v *VaultSealWatcher) isAllNodesSealed(vaultClients []*client.VaultClient) } else { v.log.Info("vault node %s is sealed", v.conf.Address) } - status = res } return status, nil From 60b17e4e3ed2d011f755398416f068463124c49f Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 26 Sep 2023 18:46:44 +0530 Subject: [PATCH 59/64] Refactored --- internal/client/vault_seal.go | 8 ++++ internal/job/vault_seal_watcher.go | 67 +++++++++--------------------- 2 files changed, 27 insertions(+), 48 deletions(-) diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index f7a4e6f8..d9abc374 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -96,6 +96,14 @@ func (vc *VaultClient) generateUnsealKeys() ([]string, string, error) { return unsealKeys, rootToken, err } +func (vc *VaultClient) Leader() (string, error) { + res, err := vc.c.Sys().Leader() + if err != nil { + return "", err + } + return res.LeaderAddress, nil +} + func (vc *VaultClient) getVaultSecretValues() (string, []string, error) { k8s, err := NewK8SClient(vc.log) if err != nil { diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 9f03503a..12439a0f 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -8,6 +8,10 @@ import ( "github.com/intelops/vault-cred/internal/client" ) +// this var is set to true after leader created for the very first time +// remove this later, after learning the correct usecase of Leader() api +var leaderCreated bool + type VaultSealWatcher struct { log logging.Logger conf config.VaultEnv @@ -80,71 +84,38 @@ func (v *VaultSealWatcher) handleUnsealForNonHAVault() error { } func (v *VaultSealWatcher) handleUnsealForHAVault() error { - var followerClients []*client.VaultClient - var leaderVaultClients []*client.VaultClient - for index, nodeAddress := range v.conf.NodeAddresses { + var vaultClients []*client.VaultClient + for _, nodeAddress := range v.conf.NodeAddresses { conf := v.conf conf.Address = nodeAddress vc, err := client.NewVaultClient(v.log, conf) if err != nil { return err } - // assuming that first node address is the leader - // Todo: keep leaders and followers in the configuration separate - if index == 0 { - leaderVaultClients = append(leaderVaultClients, vc) - } else { - followerClients = append(followerClients, vc) - } + vaultClients = append(vaultClients, vc) } - if leaderSealed, err := v.areNodesSealed(leaderVaultClients); err != nil { - return err - } else if leaderSealed { - v.log.Info("vault is sealed for leader nodes") - for _, vc := range leaderVaultClients { - err = vc.Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault for leader node, %v", err) - } + leaderNode := v.conf.LeaderAPIAddr + for _, vc := range vaultClients { + if leader, err := vc.Leader(); err == nil && leader != "" { + leaderNode = leader } } - if followersSealed, err := v.areNodesSealed(followerClients); err != nil { - return err - } else if followersSealed { - for index, vc := range followerClients { - err = vc.JoinRaftCluster(v.conf.LeaderAPIAddr) + for index, vc := range vaultClients { + if leaderCreated { + err := vc.JoinRaftCluster(leaderNode) if err != nil { return fmt.Errorf("failed to join the HA cluster by node index: %v, %v", index+1, err) } - - err = vc.Unseal() - if err != nil { - return fmt.Errorf("failed to unseal vault for node index: %v, %v", index+1, err) - } } - } - - return nil -} - -func (v *VaultSealWatcher) areNodesSealed(vaultClients []*client.VaultClient) (bool, error) { - status := false - for _, vc := range vaultClients { - res, err := vc.IsVaultSealed() - v.log.Info("unseal status for %v is %v", vc, res) + err := vc.Unseal() if err != nil { - return false, fmt.Errorf("failed to get vault seal status for %s, %v", v.conf.Address, err) - } - - if !res { - return false, nil - } else { - v.log.Info("vault node %s is sealed", v.conf.Address) + return fmt.Errorf("failed to unseal vault for node index: %v, %v", index+1, err) } - status = res + leaderCreated = true } - return status, nil + + return nil } From 4494ee4188fe85f25060274892d49385fa1efaa9 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 26 Sep 2023 19:16:15 +0530 Subject: [PATCH 60/64] Refactored Seal Watcher --- internal/job/vault_seal_watcher.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 12439a0f..83de792d 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -95,14 +95,13 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { vaultClients = append(vaultClients, vc) } - leaderNode := v.conf.LeaderAPIAddr - for _, vc := range vaultClients { + for index, vc := range vaultClients { + leaderNode := v.conf.LeaderAPIAddr + if leader, err := vc.Leader(); err == nil && leader != "" { leaderNode = leader } - } - for index, vc := range vaultClients { if leaderCreated { err := vc.JoinRaftCluster(leaderNode) if err != nil { From 7a4283eda4782bd6d2683dcb122b7c4ea5f32233 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 26 Sep 2023 19:37:57 +0530 Subject: [PATCH 61/64] Refactored Seal Watcher --- internal/job/vault_seal_watcher.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 83de792d..0f451519 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -95,24 +95,28 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { vaultClients = append(vaultClients, vc) } - for index, vc := range vaultClients { - leaderNode := v.conf.LeaderAPIAddr - + leaderNode := v.conf.LeaderAPIAddr + for _, vc := range vaultClients { if leader, err := vc.Leader(); err == nil && leader != "" { leaderNode = leader } + } + v.log.Info("Found leader node: %v", leaderNode) + for index, vc := range vaultClients { if leaderCreated { err := vc.JoinRaftCluster(leaderNode) if err != nil { - return fmt.Errorf("failed to join the HA cluster by node index: %v, %v", index+1, err) + return fmt.Errorf("failed to join the HA cluster by node index: %v, error: %v", index, err) } + v.log.Info("Node-%v successfully joined leader: %v", index, leaderNode) } err := vc.Unseal() if err != nil { - return fmt.Errorf("failed to unseal vault for node index: %v, %v", index+1, err) + return fmt.Errorf("failed to unseal vault for node index: %v, error: %v", index, err) } + v.log.Info("Node-%v successfully Unsealed", index) leaderCreated = true } From 3f46f8bd4e01b5133fd3f912d14e3dafaea7052f Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Tue, 26 Sep 2023 20:35:52 +0530 Subject: [PATCH 62/64] Refactored Seal Watcher --- internal/job/vault_seal_watcher.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index 0f451519..eabb14cf 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -83,6 +83,7 @@ func (v *VaultSealWatcher) handleUnsealForNonHAVault() error { return nil } + func (v *VaultSealWatcher) handleUnsealForHAVault() error { var vaultClients []*client.VaultClient for _, nodeAddress := range v.conf.NodeAddresses { @@ -99,6 +100,7 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { for _, vc := range vaultClients { if leader, err := vc.Leader(); err == nil && leader != "" { leaderNode = leader + leaderCreated = true } } v.log.Info("Found leader node: %v", leaderNode) @@ -112,8 +114,7 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { v.log.Info("Node-%v successfully joined leader: %v", index, leaderNode) } - err := vc.Unseal() - if err != nil { + if err := vc.Unseal(); err != nil { return fmt.Errorf("failed to unseal vault for node index: %v, error: %v", index, err) } v.log.Info("Node-%v successfully Unsealed", index) From 9e3f01e961f521f5fd923c5d77c801286bcfc576 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Fri, 29 Sep 2023 17:39:29 +0530 Subject: [PATCH 63/64] Leader address changed --- config/config.go | 2 +- internal/job/vault_seal_watcher.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 372e3ab6..db011985 100644 --- a/config/config.go +++ b/config/config.go @@ -18,7 +18,7 @@ type VaultEnv struct { HAEnabled bool `envconfig:"HA_ENABLED" default:"true"` Address string `envconfig:"VAULT_ADDR" required:"true"` NodeAddresses []string `envconfig:"VAULT_NODE_ADDRESSES" required:"true"` - LeaderAPIAddr string `envconfig:"VAULT_LEADER_ADDRESS" required:"true"` +// LeaderAPIAddr string `envconfig:"VAULT_LEADER_ADDRESS" required:"true"` CACert string `envconfig:"VAULT_CACERT" required:"false"` ReadTimeout time.Duration `envconfig:"VAULT_READ_TIMEOUT" default:"60s"` MaxRetries int `envconfig:"VAULT_MAX_RETRIES" default:"5"` diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index eabb14cf..d83851f3 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -86,6 +86,7 @@ func (v *VaultSealWatcher) handleUnsealForNonHAVault() error { func (v *VaultSealWatcher) handleUnsealForHAVault() error { var vaultClients []*client.VaultClient + var leaderNode string for _, nodeAddress := range v.conf.NodeAddresses { conf := v.conf conf.Address = nodeAddress @@ -96,7 +97,7 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { vaultClients = append(vaultClients, vc) } - leaderNode := v.conf.LeaderAPIAddr + for _, vc := range vaultClients { if leader, err := vc.Leader(); err == nil && leader != "" { leaderNode = leader From 7ddb4cfbafd0beaa4e877924fb64e7a868935447 Mon Sep 17 00:00:00 2001 From: Shifna12Zarnaz Date: Sat, 30 Sep 2023 00:16:49 +0530 Subject: [PATCH 64/64] Unseal handling for valut HA --- charts/vault-cred/templates/deployment.yaml | 4 +- charts/vault-cred/values.yaml | 2 +- config/config.go | 1 - internal/client/vault_seal.go | 1 - internal/job/vault_seal_watcher.go | 44 ++++++++++++++++----- 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/charts/vault-cred/templates/deployment.yaml b/charts/vault-cred/templates/deployment.yaml index 58fcce8a..bb24a571 100644 --- a/charts/vault-cred/templates/deployment.yaml +++ b/charts/vault-cred/templates/deployment.yaml @@ -43,12 +43,10 @@ spec: value: "{{ .Values.env.logLevel }}" - name: VAULT_ADDR value: "{{ .Values.vault.vaultAddress }}" - - name: VAULT_LEADER_ADDRESS - value: "{{ .Values.vault.vaultLeaderAddress }}" - name: VAULT_NODE_ADDRESSES value: "{{ .Values.vault.vaultNodeAddresses }}" - name: HA_ENABLED - value: "{{ .Values.vault.haenabled }}" + value: "{{ .Values.vault.haEnabled }}" - name: VAULT_SECRET_NAME value: "{{ .Values.vault.secretName }}" - name: VAULT_SECRET_TOKEN_KEY_NAME diff --git a/charts/vault-cred/values.yaml b/charts/vault-cred/values.yaml index a2cd113c..e177edba 100644 --- a/charts/vault-cred/values.yaml +++ b/charts/vault-cred/values.yaml @@ -44,7 +44,7 @@ env: logLevel: info vault: - haenabled: true + haEnabled: true vaultAddress: http://vault-hash:8200 vaultLeaderAddress: vault-hash-0.vault-hash-internal:8200 vaultNodeAddresses: "http://vault-hash-0:8200,http://vault-hash-1:8200,http://vault-hash-2:8200" diff --git a/config/config.go b/config/config.go index db011985..81edd0d5 100644 --- a/config/config.go +++ b/config/config.go @@ -18,7 +18,6 @@ type VaultEnv struct { HAEnabled bool `envconfig:"HA_ENABLED" default:"true"` Address string `envconfig:"VAULT_ADDR" required:"true"` NodeAddresses []string `envconfig:"VAULT_NODE_ADDRESSES" required:"true"` -// LeaderAPIAddr string `envconfig:"VAULT_LEADER_ADDRESS" required:"true"` CACert string `envconfig:"VAULT_CACERT" required:"false"` ReadTimeout time.Duration `envconfig:"VAULT_READ_TIMEOUT" default:"60s"` MaxRetries int `envconfig:"VAULT_MAX_RETRIES" default:"5"` diff --git a/internal/client/vault_seal.go b/internal/client/vault_seal.go index d9abc374..d76f7294 100644 --- a/internal/client/vault_seal.go +++ b/internal/client/vault_seal.go @@ -2,7 +2,6 @@ package client import ( "context" - //"encoding/base64" "fmt" "strings" diff --git a/internal/job/vault_seal_watcher.go b/internal/job/vault_seal_watcher.go index d83851f3..84fa3fca 100644 --- a/internal/job/vault_seal_watcher.go +++ b/internal/job/vault_seal_watcher.go @@ -83,10 +83,8 @@ func (v *VaultSealWatcher) handleUnsealForNonHAVault() error { return nil } - func (v *VaultSealWatcher) handleUnsealForHAVault() error { var vaultClients []*client.VaultClient - var leaderNode string for _, nodeAddress := range v.conf.NodeAddresses { conf := v.conf conf.Address = nodeAddress @@ -97,30 +95,56 @@ func (v *VaultSealWatcher) handleUnsealForHAVault() error { vaultClients = append(vaultClients, vc) } + sealed, err := v.isAnyNodeSealed(vaultClients) + if err != nil { + return err + } + + if !sealed { + v.log.Debug("All nodes are unsealed") + return nil + } + var leaderNode string for _, vc := range vaultClients { if leader, err := vc.Leader(); err == nil && leader != "" { leaderNode = leader - leaderCreated = true + break } } - v.log.Info("Found leader node: %v", leaderNode) + v.log.Infof("Found leader node: %v", leaderNode) for index, vc := range vaultClients { - if leaderCreated { + if len(leaderNode) > 0 { err := vc.JoinRaftCluster(leaderNode) if err != nil { return fmt.Errorf("failed to join the HA cluster by node index: %v, error: %v", index, err) } - v.log.Info("Node-%v successfully joined leader: %v", index, leaderNode) + v.log.Info("Node %s joined leader %s", v.conf.Address, leaderNode) } if err := vc.Unseal(); err != nil { - return fmt.Errorf("failed to unseal vault for node index: %v, error: %v", index, err) + return fmt.Errorf("failed to unseal vault on node %s, %v", v.conf.Address, err) } - v.log.Info("Node-%v successfully Unsealed", index) - leaderCreated = true - } + v.log.Info("Node %s successfully Unsealed", v.conf.Address) + } return nil } + +func (v *VaultSealWatcher) isAnyNodeSealed(vaultClients []*client.VaultClient) (bool, error) { + sealedStatus := false + for _, vc := range vaultClients { + res, err := vc.IsVaultSealed() + if err != nil { + return false, fmt.Errorf("failed to get vault seal status for %s, %v", v.conf.Address, err) + } + + if res { + sealedStatus = true + } + + v.log.Debugf("vault node %s seal status %s", v.conf.Address, res) + } + return sealedStatus, nil +}