From da4bdc652c8f2a8379905c68eede6ee86f16d3be Mon Sep 17 00:00:00 2001 From: Joe Webster <31218426+jwebster7@users.noreply.github.com> Date: Tue, 20 Aug 2024 17:12:01 -0500 Subject: [PATCH] Adds fault-injection framework This commit adds a configurable and dynamic fault-injection framework to Trident. --- frontend/csi/node_server.go | 7 + internal/fiji/fake.go | 51 +++ internal/fiji/fake_test.go | 31 ++ internal/fiji/fiji.go | 93 ++++ internal/fiji/fiji_test.go | 122 +++++ internal/fiji/inject.go | 10 + internal/fiji/models/factory.go | 66 +++ internal/fiji/models/factory_test.go | 74 +++ internal/fiji/models/handlers/always.go | 26 ++ internal/fiji/models/handlers/always_test.go | 24 + .../models/handlers/error_after_n_times.go | 40 ++ .../handlers/error_after_n_times_test.go | 64 +++ .../fiji/models/handlers/error_n_times.go | 41 ++ .../models/handlers/error_n_times_test.go | 64 +++ internal/fiji/models/handlers/exit.go | 36 ++ .../models/handlers/exit_after_n_times.go | 44 ++ .../handlers/exit_after_n_times_test.go | 70 +++ internal/fiji/models/handlers/exit_test.go | 17 + internal/fiji/models/handlers/never.go | 20 + internal/fiji/models/handlers/never_test.go | 17 + internal/fiji/models/handlers/panic.go | 30 ++ internal/fiji/models/handlers/panic_test.go | 14 + internal/fiji/models/handlers/pause.go | 59 +++ internal/fiji/models/handlers/pause_test.go | 66 +++ internal/fiji/models/point.go | 73 +++ internal/fiji/models/point_test.go | 105 +++++ internal/fiji/rest/handlers.go | 161 +++++++ internal/fiji/rest/handlers_test.go | 431 ++++++++++++++++++ internal/fiji/rest/rest.go | 21 + internal/fiji/rest/router.go | 21 + internal/fiji/rest/routes.go | 49 ++ internal/fiji/rest/routes_test.go | 16 + internal/fiji/rest/server.go | 55 +++ internal/fiji/rest/server_test.go | 21 + internal/fiji/store/store.go | 112 +++++ internal/fiji/store/store_test.go | 200 ++++++++ main.go | 13 + .../mock_internal/mock_fiji/mock_injector.go | 48 ++ .../mock_fiji/mock_models/mock_handler.go | 48 ++ .../mock_fiji/mock_rest/mock_store.go | 118 +++++ .../mock_fiji/mock_store/mock_fault.go | 88 ++++ 41 files changed, 2666 insertions(+) create mode 100644 internal/fiji/fake.go create mode 100644 internal/fiji/fake_test.go create mode 100644 internal/fiji/fiji.go create mode 100644 internal/fiji/fiji_test.go create mode 100644 internal/fiji/inject.go create mode 100644 internal/fiji/models/factory.go create mode 100644 internal/fiji/models/factory_test.go create mode 100644 internal/fiji/models/handlers/always.go create mode 100644 internal/fiji/models/handlers/always_test.go create mode 100644 internal/fiji/models/handlers/error_after_n_times.go create mode 100644 internal/fiji/models/handlers/error_after_n_times_test.go create mode 100644 internal/fiji/models/handlers/error_n_times.go create mode 100644 internal/fiji/models/handlers/error_n_times_test.go create mode 100644 internal/fiji/models/handlers/exit.go create mode 100644 internal/fiji/models/handlers/exit_after_n_times.go create mode 100644 internal/fiji/models/handlers/exit_after_n_times_test.go create mode 100644 internal/fiji/models/handlers/exit_test.go create mode 100644 internal/fiji/models/handlers/never.go create mode 100644 internal/fiji/models/handlers/never_test.go create mode 100644 internal/fiji/models/handlers/panic.go create mode 100644 internal/fiji/models/handlers/panic_test.go create mode 100644 internal/fiji/models/handlers/pause.go create mode 100644 internal/fiji/models/handlers/pause_test.go create mode 100644 internal/fiji/models/point.go create mode 100644 internal/fiji/models/point_test.go create mode 100644 internal/fiji/rest/handlers.go create mode 100644 internal/fiji/rest/handlers_test.go create mode 100644 internal/fiji/rest/rest.go create mode 100644 internal/fiji/rest/router.go create mode 100644 internal/fiji/rest/routes.go create mode 100644 internal/fiji/rest/routes_test.go create mode 100644 internal/fiji/rest/server.go create mode 100644 internal/fiji/rest/server_test.go create mode 100644 internal/fiji/store/store.go create mode 100644 internal/fiji/store/store_test.go create mode 100644 mocks/mock_internal/mock_fiji/mock_injector.go create mode 100644 mocks/mock_internal/mock_fiji/mock_models/mock_handler.go create mode 100644 mocks/mock_internal/mock_fiji/mock_rest/mock_store.go create mode 100644 mocks/mock_internal/mock_fiji/mock_store/mock_fault.go diff --git a/frontend/csi/node_server.go b/frontend/csi/node_server.go index d3bfaf445..712094f31 100644 --- a/frontend/csi/node_server.go +++ b/frontend/csi/node_server.go @@ -21,6 +21,7 @@ import ( "google.golang.org/grpc/status" tridentconfig "github.com/netapp/trident/config" + "github.com/netapp/trident/internal/fiji" . "github.com/netapp/trident/logging" sa "github.com/netapp/trident/storage_attribute" "github.com/netapp/trident/utils" @@ -50,6 +51,8 @@ var ( // NVMeNamespacesFlushRetry - Non-persistent map of Namespaces to maintain the flush errors if any. // During NodeUnstageVolume, Trident shall return success after specific wait time (nvmeMaxFlushWaitDuration). NVMeNamespacesFlushRetry = make(map[string]time.Time) + + betweenAttachAndLUKSPassphrase = fiji.Register("betweenAttachAndLUKSPassphrase", "node_server") ) func attemptLock(ctx context.Context, lockContext string, lockTimeout time.Duration) bool { @@ -1194,6 +1197,10 @@ func (p *Plugin) nodeStageISCSIVolume( } if isLUKS { + if err := betweenAttachAndLUKSPassphrase.Inject(); err != nil { + return err + } + var luksDevice utils.LUKSDeviceInterface luksDevice, err = utils.NewLUKSDeviceFromMappingPath( ctx, publishInfo.DevicePath, diff --git a/internal/fiji/fake.go b/internal/fiji/fake.go new file mode 100644 index 000000000..9381327d3 --- /dev/null +++ b/internal/fiji/fake.go @@ -0,0 +1,51 @@ +//go:build !fiji + +package fiji + +type fakeInjector struct{} + +func (f *fakeInjector) Inject() error { + // Always return nil for the fake injector. + return nil +} + +// Register is the top-level entry point for all fault points in Trident. +// Fault points MUST register through this method or a fault will not be recognized for FIJI operations. +// For non-FIJI enabled builds, this should always return an Injector. +func Register(_, _ string) Injector { + return &fakeInjector{} +} + +// Frontend is a thin structure to allow build tags to control if the FIJI API server runs or not. +// If the `fiji` build tag is not specified, this will be used instead of the real fiji implementation. +type Frontend struct{} + +func NewFrontend(_ string) *Frontend { + return &Frontend{} +} + +// Activate is a top-level entry point for enabling FIJI in Trident. +// In a FIJI-enabled build, this will create an HTTP API Server to enable dynamic fault-injection. +// In a non-FIJI build, this will do nothing. +// It is expected that this will only be called once per instance of Trident. +func (f *Frontend) Activate() error { + // Do nothing + return nil +} + +// Deactivate is the top-level entry point for disabling FIJI in Trident. +// In a FIJI-enabled build, this will gracefully close the FIJI HTTP API Server. +// In a non-FIJI build, this will do nothing. +// It is expected that this will only be called once per instance of Trident. +func (f *Frontend) Deactivate() error { + // Do nothing + return nil +} + +func (f *Frontend) GetName() string { + return "" +} + +func (f *Frontend) Version() string { + return "" +} diff --git a/internal/fiji/fake_test.go b/internal/fiji/fake_test.go new file mode 100644 index 000000000..055c23fd0 --- /dev/null +++ b/internal/fiji/fake_test.go @@ -0,0 +1,31 @@ +//go:build !fiji + +package fiji + +import ( + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + . "github.com/netapp/trident/logging" +) + +func TestMain(m *testing.M) { + // Disable any standard log output + InitLogOutput(io.Discard) + os.Exit(m.Run()) +} + +func TestFake_Register(t *testing.T) { + assert.NotNil(t, Register("", ""), "expected non-nil injector") +} + +func TestFake_Frontend(t *testing.T) { + f := NewFrontend("") + assert.Nil(t, f.Activate(), "unexpected error") + assert.Nil(t, f.Deactivate(), "unexpected error") + assert.Empty(t, f.GetName(), "unexpected name") + assert.Empty(t, f.Version(), "unexpected version") +} diff --git a/internal/fiji/fiji.go b/internal/fiji/fiji.go new file mode 100644 index 000000000..137710f30 --- /dev/null +++ b/internal/fiji/fiji.go @@ -0,0 +1,93 @@ +//go:build fiji + +package fiji + +import ( + "fmt" + "regexp" + "sync" + + "github.com/netapp/trident/internal/fiji/models" + "github.com/netapp/trident/internal/fiji/rest" + "github.com/netapp/trident/internal/fiji/store" +) + +var ( + // These regex's allow us to stop faults being created with specific characters. + // Two regexes cover reserved and disallowed chars from RFC 3986 and one is a constraint to stop "_", "~", "." + // from being used for fault names. + + specialChars = regexp.MustCompile(`[_~.]`) + reservedChars = regexp.MustCompile(`[:/?#\[\]@!$&'()*+,;=]`) + disallowedChars = regexp.MustCompile(`[ \x00-\x1F\x7F"<>\\^` + "`" + `{|}]`) + + faultStore rest.FaultStore // faultStore is an in-memory singleton for managing fault points. + initStoreOnce sync.Once // initStoreOnce protects the fault store from being initialized more than once. +) + +// initFaultStore ensures it will never be initialized more than once. +func initFaultStore() { + initStoreOnce.Do(func() { + faultStore = store.NewFaultStore() + }) +} + +// isValidName fault names are used as resource identifiers, so new fault points must have a URI-compliant fault name. +// This check should stop engineers from adding faults that don't meet our requirements. +func isValidName(name string) bool { + return !(reservedChars.MatchString(name) || disallowedChars.MatchString(name) || specialChars.MatchString(name)) +} + +// Register is the top-level entry point for all fault points in Trident. +// Fault points MUST register through this method or a fault will not be recognized for FIJI operations. +func Register(name, location string) *models.FaultPoint { + if !isValidName(name) { + panic(fmt.Errorf("invalid characters in fault name \"%s\"", name)) + } + + if faultStore == nil { + initFaultStore() + } + + fault := models.NewFaultPoint(name, location) + faultStore.Add(name, fault) + return fault +} + +// Frontend is a thin structure to allow build tags to control if the FIJI API server runs or not. +// If the `fiji` build tag is specified, this will be used instead of the fake. +type Frontend struct { + server *rest.Server +} + +func NewFrontend(address string) *Frontend { + if faultStore == nil { + initFaultStore() + } + + return &Frontend{rest.NewFaultInjectionServer(address, faultStore)} +} + +// Activate is a top-level entry point for enabling FIJI in Trident. +// In a FIJI-enabled build, this will create an HTTP API Server to enable dynamic fault-injection. +// In a non-FIJI build, this will do nothing. +// It is expected that this will only be called once per instance of Trident. +func (f *Frontend) Activate() error { + return f.server.Activate() +} + +// Deactivate is the top-level entry point for disabling FIJI in Trident. +// In a FIJI-enabled build, this will gracefully close the FIJI HTTP API Server. +// In a non-FIJI build, this will do nothing. +// It is expected that this will only be called once per instance of Trident. +func (f *Frontend) Deactivate() error { + return f.server.Deactivate() +} + +func (f *Frontend) GetName() string { + return f.server.GetName() +} + +func (f *Frontend) Version() string { + return f.server.Version() +} diff --git a/internal/fiji/fiji_test.go b/internal/fiji/fiji_test.go new file mode 100644 index 000000000..1c0489640 --- /dev/null +++ b/internal/fiji/fiji_test.go @@ -0,0 +1,122 @@ +//go:build fiji + +package fiji + +import ( + "crypto/rand" + "fmt" + "io" + "math/big" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + . "github.com/netapp/trident/logging" +) + +func TestMain(m *testing.M) { + // Disable any standard log output + InitLogOutput(io.Discard) + os.Exit(m.Run()) +} + +func TestFIJI_isValidName(t *testing.T) { + t.Run("with valid input data", func(t *testing.T) { + validData := []string{ + "faultID1", + "fault-ID456", + "faultAfterAttachBeforeEnsureLUKSPassphrase", + } + + for _, data := range validData { + assert.True(t, isValidName(data)) + } + }) + + // Test validate with strings that contain disallowed and reserved URI characters. + t.Run("with invalid input data", func(t *testing.T) { + invalidData := []string{ + "faultID@!", + "fault$id", + "fault_ID<", + "fault ID", + "fault.ID", + "fault~ID", + "fault_ID456", + " ", + } + + for _, data := range invalidData { + assert.False(t, isValidName(data)) + } + }) +} + +func TestFIJI_Register(t *testing.T) { + tt := map[string]func(*testing.T){ + "with invalid name specified": func(t *testing.T) { + name := "faultID@!" + assert.Panics(t, func() { _ = Register(name, "test") }, "expected panic") + }, + "with new fault name and duplicate fault name": func(t *testing.T) { + name := "fault-ID456" + fault := Register(name, "test") + assert.NotNil(t, fault, "expected non-nil fault") + // assert.Equal(t, name, fault.Name) + assert.Panics(t, func() { _ = Register(name, "test") }, "expected panic") + }, + } + + for name, test := range tt { + t.Run(name, test) + } +} + +func TestFIJI_RegisterIsAsyncSafe(t *testing.T) { + totalRegistrations := 500 + + // Create the test cases. + type testCase struct { + name, location string + } + location := "unit_test" + tests := make([]testCase, 0, totalRegistrations) + + // Create the test case input data. + for i := 0; i < totalRegistrations; i++ { + tests = append(tests, testCase{fmt.Sprintf("fault-%v", i), location}) + } + assert.Len(t, tests, totalRegistrations) + + // Create concurrent routines with some variance in when they fire. + wg := sync.WaitGroup{} + for _, fault := range tests { + wg.Add(1) + go func() { + // Sleep for a small amount of time to increase the chance + // of overlapping calls to Register. + time.Sleep(getTestJitter(t, int64(50*time.Microsecond))) + Register(fault.name, fault.location) + wg.Done() + }() + } + wg.Wait() + + // Ensure each fault exists in the store. + for _, fault := range tests { + assert.True(t, faultStore.Exists(fault.name)) + } +} + +func getTestJitter(t *testing.T, input int64) time.Duration { + t.Helper() + + n, err := rand.Int(rand.Reader, big.NewInt(input)) + if err != nil { + t.Fatalf("failed to calculate test jitter") + } + return time.Duration(n.Int64()) +} diff --git a/internal/fiji/inject.go b/internal/fiji/inject.go new file mode 100644 index 000000000..a3e2c2a7c --- /dev/null +++ b/internal/fiji/inject.go @@ -0,0 +1,10 @@ +package fiji + +//go:generate mockgen -destination=../../mocks/mock_internal/mock_fiji/mock_injector.go github.com/netapp/trident/internal/fiji Injector + +// Injector is an abstraction of a fault point that can inject errors. +// It is up for the configuration of the fault point to dictate the behavior of this under the hood. +// NOTE: Every concrete fault must implement the Injector interface. +type Injector interface { + Inject() error +} diff --git a/internal/fiji/models/factory.go b/internal/fiji/models/factory.go new file mode 100644 index 000000000..1e800082a --- /dev/null +++ b/internal/fiji/models/factory.go @@ -0,0 +1,66 @@ +package models + +import ( + "encoding/json" + "fmt" + + "github.com/netapp/trident/internal/fiji/models/handlers" +) + +// HandlerType tells the factory which fault to create. +type HandlerType string + +func (t HandlerType) String() string { + return string(t) +} + +const ( + // Never tells the fault to never fail. + Never HandlerType = "never" + // Pause tells the fault to pause for 'n' time then resets pause to 0. + Pause HandlerType = "pause" + // Panic tells the fault to start panicking. + // This does not kill the process immediately and any deferred functions execute. + Panic HandlerType = "panic" + // Exit kills the process immediately. Deferred functions do not execute. + Exit HandlerType = "exit" + // Always tells the fault to error indefinitely. + Always HandlerType = "always" + // ErrorNTimes tells the fault to error up to 'n' times then succeed indefinitely. + ErrorNTimes HandlerType = "error-n-times" + // ErrorAfterNTimes tells the fault to error after 'n' times indefinitely. + ErrorAfterNTimes HandlerType = "error-after-n-times" + // ExitAfterNTimes tells the fault to exit the process after 'n' times. + ExitAfterNTimes HandlerType = "exit-after-n-times" +) + +// NewFaultHandlerFromModel dynamically creates a fault handler for use by a fault. +func NewFaultHandlerFromModel(model []byte) (FaultHandler, error) { + // Create a temporary structure to pick get the model name. + getter := struct{ Name string }{} + if err := json.Unmarshal(model, &getter); err != nil { + return nil, err + } + name := getter.Name + + switch HandlerType(name) { + case Never: + return handlers.NewNeverErrorHandler(model) + case Pause: + return handlers.NewPauseHandler(model) + case Panic: + return handlers.NewPanicHandler(model) + case Exit: + return handlers.NewExitHandler(model) + case Always: + return handlers.NewAlwaysErrorHandler(model) + case ErrorNTimes: + return handlers.NewErrorNTimesHandler(model) + case ErrorAfterNTimes: + return handlers.NewErrorAfterNTimesHandler(model) + case ExitAfterNTimes: + return handlers.NewExitAfterNTimesHandler(model) + } + + return nil, fmt.Errorf("invalid value \"%s\" specified for \"Name\" in handler config", name) +} diff --git a/internal/fiji/models/factory_test.go b/internal/fiji/models/factory_test.go new file mode 100644 index 000000000..1bfd96cf8 --- /dev/null +++ b/internal/fiji/models/factory_test.go @@ -0,0 +1,74 @@ +package models + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFaultType_String(t *testing.T) { + var testStr HandlerType + testStr = "never" + assert.NotEmpty(t, testStr.String()) +} + +func TestFaultHandlerFactory(t *testing.T) { + tt := map[HandlerType]struct { + model string + assertNil assert.ValueAssertionFunc + assertError assert.ErrorAssertionFunc + }{ + Never: { + model: `{"name": "never"}`, + assertNil: assert.NotNil, + assertError: assert.NoError, + }, + Panic: { + model: fmt.Sprintf(`{"name": "panic"}`), + assertNil: assert.NotNil, + assertError: assert.NoError, + }, + Pause: { + model: fmt.Sprintf(`{"name": "pause", "duration": "%s"}`, "500ms"), + assertNil: assert.NotNil, + assertError: assert.NoError, + }, + Exit: { + model: fmt.Sprintf(`{"name": "exit", "exitCode": %v}`, 1), + assertNil: assert.NotNil, + assertError: assert.NoError, + }, + Always: { + model: `{"name": "always"}`, + assertNil: assert.NotNil, + assertError: assert.NoError, + }, + ErrorNTimes: { + model: fmt.Sprintf(`{"name": "error-n-times", "failCount": %v}`, 1), + assertNil: assert.NotNil, + assertError: assert.NoError, + }, + ErrorAfterNTimes: { + model: fmt.Sprintf(`{"name": "error-after-n-times", "passCount": %v}`, 1), + assertNil: assert.NotNil, + assertError: assert.NoError, + }, + ExitAfterNTimes: { + model: fmt.Sprintf(`{"name": "exit-after-n-times", "exitCode": %v, "passCount": %v}`, 1, 1), + assertNil: assert.NotNil, + assertError: assert.NoError, + }, + } + + for name, test := range tt { + t.Run(string(name), func(t *testing.T) { + model := []byte(test.model) + assert.NotNil(t, model) + + handler, err := NewFaultHandlerFromModel(model) + test.assertNil(t, handler) + test.assertError(t, err) + }) + } +} diff --git a/internal/fiji/models/handlers/always.go b/internal/fiji/models/handlers/always.go new file mode 100644 index 000000000..9b8c9b130 --- /dev/null +++ b/internal/fiji/models/handlers/always.go @@ -0,0 +1,26 @@ +package handlers + +import ( + "encoding/json" + "fmt" + + . "github.com/netapp/trident/logging" +) + +type AlwaysErrorHandler struct { + Name string `json:"name"` +} + +func (a *AlwaysErrorHandler) Handle() error { + Log().Debugf("Injecting error from %s handler.", a.Name) + return fmt.Errorf("fiji error from %s handler", a.Name) +} + +func NewAlwaysErrorHandler(model []byte) (*AlwaysErrorHandler, error) { + var alwaysErrorHandler AlwaysErrorHandler + if err := json.Unmarshal(model, &alwaysErrorHandler); err != nil { + return nil, err + } + + return &alwaysErrorHandler, nil +} diff --git a/internal/fiji/models/handlers/always_test.go b/internal/fiji/models/handlers/always_test.go new file mode 100644 index 000000000..430a05fee --- /dev/null +++ b/internal/fiji/models/handlers/always_test.go @@ -0,0 +1,24 @@ +package handlers + +import ( + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + . "github.com/netapp/trident/logging" +) + +func TestMain(m *testing.M) { + // Disable any standard log output + InitLogOutput(io.Discard) + os.Exit(m.Run()) +} + +func TestAlwaysHandler(t *testing.T) { + handler, err := NewAlwaysErrorHandler([]byte(`{"name":"always"}`)) + assert.NoError(t, err) + assert.NotNil(t, handler) + assert.Error(t, handler.Handle()) +} diff --git a/internal/fiji/models/handlers/error_after_n_times.go b/internal/fiji/models/handlers/error_after_n_times.go new file mode 100644 index 000000000..8ea4b072a --- /dev/null +++ b/internal/fiji/models/handlers/error_after_n_times.go @@ -0,0 +1,40 @@ +package handlers + +import ( + "encoding/json" + "fmt" + + . "github.com/netapp/trident/logging" +) + +type ErrorAfterNTimesHandler struct { + Name string `json:"name"` + HitCount int `json:"hitCount"` + PassCount int `json:"passCount"` +} + +func (en *ErrorAfterNTimesHandler) Handle() error { + Log().Debugf("Firing %s handler.", en.Name) + // While the passCount is greater than the hitCount, this handler should return nil. + // When the hitCount exceeds the passCount, this handler should fail indefinitely. + en.HitCount++ + if en.HitCount <= en.PassCount { + remaining := en.PassCount - en.HitCount + Log().Debugf("%v remaining passes from %s handler.", remaining, en.Name) + return nil + } + return fmt.Errorf("fiji error from [%s] handler; infinite errors remaining", en.Name) +} + +func NewErrorAfterNTimesHandler(model []byte) (*ErrorAfterNTimesHandler, error) { + var handler ErrorAfterNTimesHandler + if err := json.Unmarshal(model, &handler); err != nil { + return nil, err + } + + if handler.PassCount <= 0 { + return nil, fmt.Errorf("invalid value specified for passCount") + } + + return &handler, nil +} diff --git a/internal/fiji/models/handlers/error_after_n_times_test.go b/internal/fiji/models/handlers/error_after_n_times_test.go new file mode 100644 index 000000000..ada068780 --- /dev/null +++ b/internal/fiji/models/handlers/error_after_n_times_test.go @@ -0,0 +1,64 @@ +package handlers + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestErrorAfterNTimesHandler(t *testing.T) { + tt := map[string]struct { + formatStr string + passCount int + assertValue assert.ValueAssertionFunc + assertError assert.ErrorAssertionFunc + }{ + "with no KVP for count": { + formatStr: `{"name":"error-after-n-times"}`, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with negative exit code": { + formatStr: `{"name":"error-after-n-times", "passCount": %v}`, + passCount: -1, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with zero exit code": { + formatStr: `{"name":"error-after-n-times", "passCount": %v}`, + passCount: 0, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with valid values": { + formatStr: `{"name":"error-after-n-times", "passCount": %v}`, + passCount: 1, + assertValue: assert.NotNil, + assertError: assert.NoError, + }, + } + + for name, test := range tt { + t.Run(name, func(t *testing.T) { + modelStr := fmt.Sprintf(test.formatStr, test.passCount) + handler, err := NewErrorAfterNTimesHandler([]byte(modelStr)) + test.assertError(t, err) + test.assertValue(t, handler) + }) + } +} + +func TestErrorAfterNTimesHandler_Handle(t *testing.T) { + count := 5 + modelJSON := fmt.Sprintf(`{"name":"error-after-n-times", "passCount": %v}`, count) + handler, err := NewErrorAfterNTimesHandler([]byte(modelJSON)) + assert.NoError(t, err) + assert.NotNil(t, handler) + assert.Equal(t, count, handler.PassCount) + + for i := 0; i < count; i++ { + assert.NoError(t, handler.Handle()) + } + assert.Error(t, handler.Handle()) +} diff --git a/internal/fiji/models/handlers/error_n_times.go b/internal/fiji/models/handlers/error_n_times.go new file mode 100644 index 000000000..c1690a9ab --- /dev/null +++ b/internal/fiji/models/handlers/error_n_times.go @@ -0,0 +1,41 @@ +package handlers + +import ( + "encoding/json" + "fmt" + + . "github.com/netapp/trident/logging" +) + +type ErrorNTimesHandler struct { + Name string `json:"name"` + HitCount int `json:"hitCount"` + FailCount int `json:"failCount"` +} + +func (fn *ErrorNTimesHandler) Handle() error { + Log().Debugf("Firing %s handler.", fn.Name) + // While the hitCount is less than the failCount, this handler should fail. + // Once the hitCount exceeds the failCount, this handler should return nil indefinitely. + if fn.HitCount < fn.FailCount { + fn.HitCount++ + remaining := fn.FailCount - fn.HitCount + Log().Debugf("%v remaining errors from %s handler.", remaining, fn.Name) + return fmt.Errorf("fiji error from [%s] handler; %v errors remaining", fn.Name, remaining) + } + Log().Debugf("No errors remaining from %s handler.", fn.Name) + return nil +} + +func NewErrorNTimesHandler(model []byte) (*ErrorNTimesHandler, error) { + var handler ErrorNTimesHandler + if err := json.Unmarshal(model, &handler); err != nil { + return nil, err + } + + if handler.FailCount <= 0 { + return nil, fmt.Errorf("invalid value specified for failCount") + } + + return &handler, nil +} diff --git a/internal/fiji/models/handlers/error_n_times_test.go b/internal/fiji/models/handlers/error_n_times_test.go new file mode 100644 index 000000000..36daf99fc --- /dev/null +++ b/internal/fiji/models/handlers/error_n_times_test.go @@ -0,0 +1,64 @@ +package handlers + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestErrorNTimesHandler(t *testing.T) { + tt := map[string]struct { + formatStr string + failCount int + assertValue assert.ValueAssertionFunc + assertError assert.ErrorAssertionFunc + }{ + "with no KVP for count": { + formatStr: `{"name":"error-n-times"}`, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with negative value for count": { + formatStr: `{"name":"error-n-times", "failCount": %v}`, + failCount: -1, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with zero value for count": { + formatStr: `{"name":"error-n-times", "failCount": %v}`, + failCount: 0, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with positive value for count": { + formatStr: `{"name":"error-n-times", "failCount": %v}`, + failCount: 1, + assertValue: assert.NotNil, + assertError: assert.NoError, + }, + } + + for name, test := range tt { + t.Run(name, func(t *testing.T) { + modelStr := fmt.Sprintf(test.formatStr, test.failCount) + handler, err := NewErrorNTimesHandler([]byte(modelStr)) + test.assertError(t, err) + test.assertValue(t, handler) + }) + } +} + +func TestErrorNTimesHandler_Handle(t *testing.T) { + count := 5 + modelJSON := fmt.Sprintf(`{"name":"error-n-times", "failCount": %v}`, count) + handler, err := NewErrorNTimesHandler([]byte(modelJSON)) + assert.NoError(t, err) + assert.NotNil(t, handler) + assert.Equal(t, count, handler.FailCount) + + for i := 0; i < count; i++ { + assert.Error(t, handler.Handle()) + } + assert.NoError(t, handler.Handle()) +} diff --git a/internal/fiji/models/handlers/exit.go b/internal/fiji/models/handlers/exit.go new file mode 100644 index 000000000..707698207 --- /dev/null +++ b/internal/fiji/models/handlers/exit.go @@ -0,0 +1,36 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "os" + + . "github.com/netapp/trident/logging" +) + +type ExitHandler struct { + Name string `json:"name"` + ExitCode int `json:"exitCode"` +} + +func (e *ExitHandler) Handle() error { + Log().Debugf("Firing %s handler", e.Name) + if e.ExitCode >= 0 { + Log().Debugf("Injecting exit with code %v from %s handler.", e.ExitCode, e.Name) + os.Exit(e.ExitCode) + } + return nil +} + +func NewExitHandler(model []byte) (*ExitHandler, error) { + var handler ExitHandler + if err := json.Unmarshal(model, &handler); err != nil { + return nil, err + } + + if handler.ExitCode < 0 { + return nil, fmt.Errorf("invalid value specified for exitCode") + } + + return &handler, nil +} diff --git a/internal/fiji/models/handlers/exit_after_n_times.go b/internal/fiji/models/handlers/exit_after_n_times.go new file mode 100644 index 000000000..8cab1a618 --- /dev/null +++ b/internal/fiji/models/handlers/exit_after_n_times.go @@ -0,0 +1,44 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "os" + + . "github.com/netapp/trident/logging" +) + +type ExitAfterNTimesHandler struct { + Name string `json:"name"` + ExitCode int `json:"exitCode"` + HitCount int `json:"hitCount"` + PassCount int `json:"passCount"` +} + +func (ent *ExitAfterNTimesHandler) Handle() error { + Log().Debugf("Firing %s handler.", ent.Name) + ent.HitCount++ + if ent.PassCount <= ent.HitCount { + Log().Debugf("Injecting exit from %s handler.", ent.Name) + os.Exit(ent.ExitCode) + } + remaining := ent.PassCount - ent.HitCount + Log().Debugf("%v remaining passes from %s handler.", remaining, ent.Name) + return nil +} + +func NewExitAfterNTimesHandler(model []byte) (*ExitAfterNTimesHandler, error) { + var handler ExitAfterNTimesHandler + if err := json.Unmarshal(model, &handler); err != nil { + return nil, err + } + + if handler.ExitCode < 0 { + return nil, fmt.Errorf("invalid value specified for exitCode") + } + if handler.PassCount <= 0 { + return nil, fmt.Errorf("invalid value specified for passCount") + } + + return &handler, nil +} diff --git a/internal/fiji/models/handlers/exit_after_n_times_test.go b/internal/fiji/models/handlers/exit_after_n_times_test.go new file mode 100644 index 000000000..d9c7bb1c4 --- /dev/null +++ b/internal/fiji/models/handlers/exit_after_n_times_test.go @@ -0,0 +1,70 @@ +package handlers + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExitAfterNTimesHandler(t *testing.T) { + tt := map[string]struct { + formatStr string + passCount int + exitCode int + assertValue assert.ValueAssertionFunc + assertError assert.ErrorAssertionFunc + }{ + "with no KVP for count specified": { + formatStr: `{"name":"exit-after-n-times"}`, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with invalid type for count specified": { + formatStr: `{"name":"exit-after-n-times", "passCount": "test"}`, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with invalid type fo exit code specified": { + formatStr: `{"name":"exit-after-n-times", "exitCode": "1"}`, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with invalid type for exit code specified": { + formatStr: `{"name":"exit-after-n-times", "passCount": %v,"exitCode": "one"}`, + passCount: 1, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with negative pass count configured": { + formatStr: `{"name":"exit-after-n-times", "passCount": %v,"exitCode": %v}`, + passCount: -1, + exitCode: 1, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with negative exit code configured": { + formatStr: `{"name":"exit-after-n-times", "passCount": %v,"exitCode": %v}`, + passCount: 1, + exitCode: -1, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with valid values": { + formatStr: `{"name":"exit-after-n-times", "passCount": %v,"exitCode": %v}`, + passCount: 1, + exitCode: 0, + assertValue: assert.NotNil, + assertError: assert.NoError, + }, + } + + for name, test := range tt { + t.Run(name, func(t *testing.T) { + modelStr := fmt.Sprintf(test.formatStr, test.passCount, test.exitCode) + handler, err := NewExitAfterNTimesHandler([]byte(modelStr)) + test.assertError(t, err) + test.assertValue(t, handler) + }) + } +} diff --git a/internal/fiji/models/handlers/exit_test.go b/internal/fiji/models/handlers/exit_test.go new file mode 100644 index 000000000..f2f7046fa --- /dev/null +++ b/internal/fiji/models/handlers/exit_test.go @@ -0,0 +1,17 @@ +package handlers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExitHandler(t *testing.T) { + handler, err := NewExitHandler([]byte(`{"name":"exit", "exitCode": -1}`)) + assert.Error(t, err) + assert.Nil(t, handler) + + handler, err = NewExitHandler([]byte(`{"name":"exit", "exitCode": 1}`)) + assert.NoError(t, err) + assert.NotNil(t, handler) +} diff --git a/internal/fiji/models/handlers/never.go b/internal/fiji/models/handlers/never.go new file mode 100644 index 000000000..5c775d62c --- /dev/null +++ b/internal/fiji/models/handlers/never.go @@ -0,0 +1,20 @@ +package handlers + +import "encoding/json" + +type NeverErrorHandler struct { + Name string `json:"name"` +} + +func (n *NeverErrorHandler) Handle() error { + return nil +} + +func NewNeverErrorHandler(model []byte) (*NeverErrorHandler, error) { + var handler NeverErrorHandler + if err := json.Unmarshal(model, &handler); err != nil { + return nil, err + } + + return &handler, nil +} diff --git a/internal/fiji/models/handlers/never_test.go b/internal/fiji/models/handlers/never_test.go new file mode 100644 index 000000000..c50d7477f --- /dev/null +++ b/internal/fiji/models/handlers/never_test.go @@ -0,0 +1,17 @@ +package handlers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNeverErrorHandler(t *testing.T) { + neverName := "never" + handler, err := NewNeverErrorHandler([]byte(`{"name": "never"}`)) + assert.NoError(t, err) + assert.NotNil(t, handler) + + assert.Equal(t, neverName, handler.Name) + assert.NoError(t, handler.Handle()) +} diff --git a/internal/fiji/models/handlers/panic.go b/internal/fiji/models/handlers/panic.go new file mode 100644 index 000000000..0054b0dad --- /dev/null +++ b/internal/fiji/models/handlers/panic.go @@ -0,0 +1,30 @@ +package handlers + +import ( + "encoding/json" + "fmt" + + . "github.com/netapp/trident/logging" +) + +type PanicHandler struct { + Name string `json:"name"` +} + +func (p *PanicHandler) Handle() (err error) { + Log().Debugf("Firing %s handler.", p.Name) + defer func() { + Log().Debugf("Injecting panic from %s handler.", p.Name) + panic(fmt.Errorf("panic from fiji handler %s", p.Name)) + }() + return +} + +func NewPanicHandler(model []byte) (*PanicHandler, error) { + var handler PanicHandler + if err := json.Unmarshal(model, &handler); err != nil { + return nil, err + } + + return &handler, nil +} diff --git a/internal/fiji/models/handlers/panic_test.go b/internal/fiji/models/handlers/panic_test.go new file mode 100644 index 000000000..bbcf94898 --- /dev/null +++ b/internal/fiji/models/handlers/panic_test.go @@ -0,0 +1,14 @@ +package handlers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPanicHandler(t *testing.T) { + handler, err := NewPanicHandler([]byte(`{"name":"panic"}`)) + assert.NoError(t, err) + assert.NotNil(t, handler) + assert.Panics(t, func() { handler.Handle() }) +} diff --git a/internal/fiji/models/handlers/pause.go b/internal/fiji/models/handlers/pause.go new file mode 100644 index 000000000..4b1599257 --- /dev/null +++ b/internal/fiji/models/handlers/pause.go @@ -0,0 +1,59 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "time" + + . "github.com/netapp/trident/logging" +) + +type PauseHandler struct { + Name string `json:"name"` + Pause pause `json:"duration"` +} + +func (p *PauseHandler) Handle() error { + Log().Debugf("Injecting sleep for %s from %s handler.", p.Pause.String(), p.Name) + time.Sleep(p.Pause.Abs()) + return nil +} + +func NewPauseHandler(model []byte) (*PauseHandler, error) { + var handler PauseHandler + if err := json.Unmarshal(model, &handler); err != nil { + return nil, err + } + + return &handler, nil +} + +type pause struct { + time.Duration +} + +func (p *pause) MarshalJSON() ([]byte, error) { + return json.Marshal(p.String()) +} + +// UnmarshalJSON allows custom unmarshalling so that "duration" may be specified as an amount of time with units. +// Example: "duration": "500ms". +func (p *pause) UnmarshalJSON(b []byte) error { + // Check if nothing was supplied or if an empty string was. + if len(b) == 0 || len(b) == 2 { + return fmt.Errorf("invalid value specified for duration") + } + + duration, err := time.ParseDuration(string(b[1 : len(b)-1])) + if err != nil { + return err + } + + if duration <= 0 { + return fmt.Errorf("invalid value specified for duration") + } + + // Set the duration + p.Duration = duration + return nil +} diff --git a/internal/fiji/models/handlers/pause_test.go b/internal/fiji/models/handlers/pause_test.go new file mode 100644 index 000000000..77ea7ff58 --- /dev/null +++ b/internal/fiji/models/handlers/pause_test.go @@ -0,0 +1,66 @@ +package handlers + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestPauseHandler(t *testing.T) { + tt := map[string]struct { + formatStr string + duration time.Duration + assertValue assert.ValueAssertionFunc + assertError assert.ErrorAssertionFunc + }{ + "with no KVP for duration": { + formatStr: `{"name":"pause"}`, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with negative duration": { + formatStr: `{"name":"pause", "duration": "%s"}`, + duration: -500 * time.Millisecond, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with zero exit code": { + formatStr: `{"name":"pause", "duration": "%s"}`, + assertValue: assert.Nil, + assertError: assert.Error, + }, + "with valid values": { + formatStr: `{"name":"pause", "duration": "%s"}`, + duration: 500 * time.Millisecond, + assertValue: assert.NotNil, + assertError: assert.NoError, + }, + } + + for name, test := range tt { + t.Run(name, func(t *testing.T) { + modelStr := fmt.Sprintf(test.formatStr, test.duration) + handler, err := NewPauseHandler([]byte(modelStr)) + test.assertError(t, err) + test.assertValue(t, handler) + }) + } +} + +func TestPauseHandler_Handle(t *testing.T) { + handler, err := NewPauseHandler([]byte(`{"name":"pause", "duration": "500ms"}`)) + assert.NoError(t, err) + assert.NotNil(t, handler) + + maximumPauseTime, err := time.ParseDuration("500ms") + assert.NoError(t, err) + + start := time.Now() + _ = handler.Handle() + stop := time.Since(start) + // Add a padding to handle variability between runs. + maximumPauseTime += 5000 * time.Microsecond + assert.GreaterOrEqual(t, maximumPauseTime, stop) +} diff --git a/internal/fiji/models/point.go b/internal/fiji/models/point.go new file mode 100644 index 000000000..fef9b048f --- /dev/null +++ b/internal/fiji/models/point.go @@ -0,0 +1,73 @@ +package models + +//go:generate mockgen -destination=../../../mocks/mock_internal/mock_fiji/mock_models/mock_handler.go github.com/netapp/trident/internal/fiji/models FaultHandler + +import ( + "fmt" + "sync" +) + +// FaultHandler is an abstraction all concrete fault handler should follow. +type FaultHandler interface { + Handle() error +} + +// FaultPoint is a concrete realization of the Fault interface. +type FaultPoint struct { + mutex *sync.RWMutex + Name string `json:"name"` + Location string `json:"location"` + Handler FaultHandler `json:"handler,omitempty"` +} + +// NewFaultPoint creates a new fault point with a default handler that never injects an error. +func NewFaultPoint(name, location string) *FaultPoint { + return &FaultPoint{ + Name: name, + Location: location, + mutex: &sync.RWMutex{}, + } +} + +func (f *FaultPoint) Inject() error { + f.mutex.Lock() + defer f.mutex.Unlock() + + // Do nothing if this fault handler hasn't been set. + if f.Handler == nil { + return nil + } + return f.Handler.Handle() +} + +func (f *FaultPoint) Reset() { + f.mutex.Lock() + defer f.mutex.Unlock() + + f.Handler = nil + return +} + +func (f *FaultPoint) IsHandlerSet() bool { + f.mutex.RLock() + defer f.mutex.RUnlock() + + return f.Handler != nil +} + +func (f *FaultPoint) SetHandler(config []byte) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + if config == nil { + return fmt.Errorf("empty model supplied for fault: [%s]", f.Name) + } + + handler, err := NewFaultHandlerFromModel(config) + if err != nil { + return fmt.Errorf("failed to determine new fault handler; %v", err) + } + + f.Handler = handler + return nil +} diff --git a/internal/fiji/models/point_test.go b/internal/fiji/models/point_test.go new file mode 100644 index 000000000..073ff91df --- /dev/null +++ b/internal/fiji/models/point_test.go @@ -0,0 +1,105 @@ +package models + +import ( + "fmt" + "io" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + . "github.com/netapp/trident/logging" +) + +func TestMain(m *testing.M) { + // Disable any standard log output + InitLogOutput(io.Discard) + os.Exit(m.Run()) +} + +type fakeStore struct { + lock *sync.RWMutex + data map[string]*FaultPoint +} + +func newFakeStore(faults ...*FaultPoint) *fakeStore { + data := make(map[string]*FaultPoint, len(faults)) + for _, fault := range faults { + data[fault.Name] = fault + } + + return &fakeStore{ + lock: &sync.RWMutex{}, data: data, + } +} + +func (s *fakeStore) Update(name string, model []byte) error { + s.lock.Lock() + defer s.lock.Unlock() + + fault, ok := s.data[name] + if !ok { + return fmt.Errorf("test fault \"%s\" not found", name) + } + return fault.SetHandler(model) +} + +func TestFaultPoint_Inject(t *testing.T) { + point := NewFaultPoint("name", "unit_test") + assert.NoError(t, point.Inject()) +} + +func TestFaultPoint_SetHandler(t *testing.T) { + point := NewFaultPoint("name", "unit_test") + assert.Error(t, point.SetHandler(nil)) + + model := `{"name":"pause", "duration": 300}` + err := point.SetHandler([]byte(model)) + assert.Error(t, err) + + model = `{"name":"pause","duration":"300ms"}` + err = point.SetHandler([]byte(model)) + assert.NoError(t, err) +} + +// TestFaultPoint_SetHandlerConfig_Async tests that it is safe to use a mutex lock on the fault points +// when consumers try to modify it's internal state. Consumers will have to wait for the +// current inject() call to finish before being able to modify the handler config. +func TestFaultPoint_SetHandler_Async(t *testing.T) { + point := NewFaultPoint("name", "unit_test") + + // Create a store that has access to the fault point above and a mutex. + store := newFakeStore(point) + start := time.Now() + duration := 100 * time.Millisecond + modelStr := fmt.Sprintf(`{"name": "pause", "duration": "%s"}`, duration.String()) + err := point.SetHandler([]byte(modelStr)) + assert.NoError(t, err) + + // This starts a routine that will "inject" a pause. + var errFromSleep error + go func() { + // This should induce a sleep for the length of "duration" that holds the point's mutex. + errFromSleep = point.Inject() + }() + // Add a short sleep here to ensure the go routine has time to + // call "point.Inject" before resetting the handler config. + time.Sleep(25 * time.Millisecond) + + // The points original handler will sleep, causing the points mutex to be in use. + err = store.Update(point.Name, []byte(`{"name": "always"}`)) + assert.NoError(t, err) + + // This assertion ensures the store has to wait for the points mutex to unlock before changing the handler. + after := time.Since(start) + assert.GreaterOrEqual(t, after, duration) + afterChangingHandler := time.Now() + + // Ensure the new handler doesn't sleep. + err = point.Inject() + assert.LessOrEqual(t, time.Since(afterChangingHandler), duration) + assert.NotNil(t, err) + assert.NotEqual(t, err, errFromSleep) +} diff --git a/internal/fiji/rest/handlers.go b/internal/fiji/rest/handlers.go new file mode 100644 index 000000000..8b629818c --- /dev/null +++ b/internal/fiji/rest/handlers.go @@ -0,0 +1,161 @@ +package rest + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + + "github.com/gorilla/mux" +) + +// makeListFaultsHandler lists ALL fault configuration that exist. +func makeListFaultsHandler(store FaultStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + var enabled *bool + + // Check if enabled param is present as a route variable. If it is, then use it. + params := r.URL.Query() + if params.Get("enabled") != "" { + // Enabled represents 3 states: set / not set, true, and false. + // Therefore, it should only be set if the query param exists. + enabledSet, err := strconv.ParseBool(params.Get("enabled")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + enabled = &enabledSet + } + + // Build a response body. + faults := store.List() + externalFaults := make([]any, 0, len(faults)) + for _, fault := range faults { + // If the enabled query param isn't set, add every fault to the list. + if enabled == nil { + externalFaults = append(externalFaults, fault) + continue + } + + // Add the current fault to the list if its enabled state equals the query params value. + if *enabled == fault.IsHandlerSet() { + externalFaults = append(externalFaults, fault) + continue + } + } + + // Get the faults stored in the store and convert them to JSON. + data, err := json.Marshal(externalFaults) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + if _, err := w.Write(data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + return + } +} + +// makeGetFaultHandler retrieves a single fault configuration that exists. +func makeGetFaultHandler(store FaultStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + + // Check if the name is present as a route variable. If it is, then use it. + name, ok := mux.Vars(r)["name"] + if !ok { + http.Error(w, "no fault config specified", http.StatusBadRequest) + return + } + + fault, exists := store.Get(name) + if !exists || fault == nil { + http.Error(w, "no fault found", http.StatusNotFound) + return + } + + // Get the faults stored in the store and convert them to JSON. + data, err := json.Marshal(fault) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + if _, err := w.Write(data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + return + } +} + +// makeSetFaultConfigHandler attempts to put a single fault configuration and handler for a registered fault. +func makeSetFaultConfigHandler(store FaultStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + + name, ok := mux.Vars(r)["name"] + if !ok { + http.Error(w, "no fault name specified", http.StatusBadRequest) + return + } + + if exists := store.Exists(name); !exists { + http.Error(w, fmt.Sprintf("specified fault [%s] does not exist", name), http.StatusNotFound) + return + } + + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } else if len(body) == 0 { + http.Error(w, "empty request body", http.StatusBadRequest) + return + } + + if err = store.Set(name, body); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusAccepted) + return + } +} + +// makeResetFaultConfigHandler attempts to reset a single faults handler config to its original state. +func makeResetFaultConfigHandler(store FaultStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + + name, ok := mux.Vars(r)["name"] + if !ok || name == "" { + http.Error(w, "no fault name specified", http.StatusBadRequest) + return + } + + if exists := store.Exists(name); !exists { + http.Error(w, fmt.Sprintf("specified fault [%s] does not exist", name), http.StatusNotFound) + return + } + + // Ignore any payload and just reset the fault. + if err := store.Reset(name); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusAccepted) + return + } +} diff --git a/internal/fiji/rest/handlers_test.go b/internal/fiji/rest/handlers_test.go new file mode 100644 index 000000000..b5d9eb4f1 --- /dev/null +++ b/internal/fiji/rest/handlers_test.go @@ -0,0 +1,431 @@ +package rest + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/golang/mock/gomock" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + + "github.com/netapp/trident/internal/fiji/store" + . "github.com/netapp/trident/logging" + "github.com/netapp/trident/mocks/mock_internal/mock_fiji/mock_rest" +) + +func TestMain(m *testing.M) { + // Disable any standard log output + InitLogOutput(io.Discard) + os.Exit(m.Run()) +} + +type fakeFault struct { + Name string `json:"name,omitempty"` + Location string `json:"location,omitempty"` + Handler any `json:"handler,omitempty"` +} + +func (f *fakeFault) Inject() error { + return nil +} + +func (f *fakeFault) Reset() { + f.Handler = nil +} + +func (f *fakeFault) IsHandlerSet() bool { + return f.Handler != nil +} + +func (f *fakeFault) SetHandler(m []byte) error { + f.Handler = string(m) + return nil +} + +// newHttpTestServer sets up a httptest server with a supplied handler and pattern. +func newHttpTestServer(t *testing.T, pattern string, handler http.HandlerFunc) *httptest.Server { + t.Helper() + router := mux.NewRouter() + router.HandleFunc(pattern, handler) + return httptest.NewServer(router) +} + +func TestHandlers_makeListFaultsHandler(t *testing.T) { + mockCtrl := gomock.NewController(t) + t.Run("with 1 fault found", func(t *testing.T) { + fake := &fakeFault{ + Name: "fault", + Location: "location", + Handler: `{"name":"handler-name"}`, + } + + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + mockStore.EXPECT().List().Return([]store.Fault{fake}) + + handler := makeListFaultsHandler(mockStore) + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Get(server.URL) + assert.NotNil(t, resp, "expected non-nil response") + assert.NoError(t, err, "expected error") + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + assert.FailNow(t, err.Error()) + } + + var respBody []fakeFault + if err := json.Unmarshal(body, &respBody); err != nil { + assert.FailNow(t, err.Error()) + } + assert.Len(t, respBody, 1) + + externalFault := respBody[0] + assert.Equal(t, fake.Name, externalFault.Name) + assert.Equal(t, fake.Location, externalFault.Location) + assert.EqualValues(t, fake.Handler, externalFault.Handler) + }) + + t.Run("with 1 enabled fault found", func(t *testing.T) { + fake := &fakeFault{ + Name: "fault", + Location: "location", + Handler: `{"name":"handler-name"}`, + } + + fakeFaults := []store.Fault{ + fake, + &fakeFault{ + Name: "fault2", + Location: "location", + // model: []byte(`{"name":"handler-name"}`), + }, + } + + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + mockStore.EXPECT().List().Return(fakeFaults) + + handler := makeListFaultsHandler(mockStore) + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Get(server.URL + "?enabled=true") + assert.NotNil(t, resp, "expected non-nil response") + assert.NoError(t, err, "expected error") + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + assert.FailNow(t, err.Error()) + } + + var listFaultRespBody []fakeFault + if err := json.Unmarshal(body, &listFaultRespBody); err != nil { + assert.FailNow(t, err.Error()) + } + assert.Len(t, listFaultRespBody, 1) + + externalFault := listFaultRespBody[0] + assert.Equal(t, fake.Name, externalFault.Name) + assert.Equal(t, fake.Location, externalFault.Location) + assert.EqualValues(t, fake.Handler, externalFault.Handler) + assert.Equal(t, fake.IsHandlerSet(), externalFault.IsHandlerSet()) + }) + + t.Run("with no enabled fault found", func(t *testing.T) { + // Create fake faults with no handler specified. + fakeFaults := []store.Fault{ + &fakeFault{ + Name: "fault", + Location: "location", + }, + &fakeFault{ + Name: "fault", + Location: "location", + }, + } + + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + mockStore.EXPECT().List().Return(fakeFaults) + + handler := makeListFaultsHandler(mockStore) + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Get(server.URL + "?enabled=true") + assert.NotNil(t, resp, "expected non-nil response") + assert.NoError(t, err, "expected error") + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + assert.FailNow(t, err.Error()) + } + + var listFaultRespBody []fakeFault + if err := json.Unmarshal(body, &listFaultRespBody); err != nil { + assert.FailNow(t, err.Error()) + } + assert.Len(t, listFaultRespBody, 0) + }) +} + +func TestHandlers_makeGetFaultHandler(t *testing.T) { + mockCtrl := gomock.NewController(t) + + t.Run("with no name id specified", func(t *testing.T) { + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + server := newHttpTestServer(t, "/{name}", makeGetFaultHandler(mockStore)) + defer server.Close() + + resp, _ := http.Get(fmt.Sprintf("%s/%s", server.URL, "")) + assert.NotNil(t, resp, "expected non-nil response") + }) + + t.Run("with name id and no fault", func(t *testing.T) { + fake := &fakeFault{ + Name: "fault", + Location: "location", + Handler: `{"name":"handler-name"}`, + } + + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + mockStore.EXPECT().Get(fake.Name).Return(nil, false) + + server := newHttpTestServer(t, "/{name}", makeGetFaultHandler(mockStore)) + defer server.Close() + + resp, _ := http.Get(fmt.Sprintf("%s/%s", server.URL, fake.Name)) + assert.NotNil(t, resp, "expected non-nil response") + assert.Equal(t, http.StatusNotFound, resp.StatusCode, "expected status codes to match") + }) + + t.Run("with name id", func(t *testing.T) { + fake := &fakeFault{ + Name: "fault", + Location: "location", + Handler: `{"name":"handler-name"}`, + } + + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + mockStore.EXPECT().Get(fake.Name).Return(fake, true) + + server := newHttpTestServer(t, "/{name}", makeGetFaultHandler(mockStore)) + defer server.Close() + + resp, _ := http.Get(fmt.Sprintf("%s/%s", server.URL, fake.Name)) + assert.NotNil(t, resp, "expected non-nil response") + assert.Equal(t, http.StatusOK, resp.StatusCode, "expected status codes to match") + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + assert.FailNow(t, err.Error()) + } + + var getFaultRespBody fakeFault + if err := json.Unmarshal(body, &getFaultRespBody); err != nil { + assert.FailNow(t, err.Error()) + } + + assert.Equal(t, fake.Name, getFaultRespBody.Name) + assert.Equal(t, fake.Location, getFaultRespBody.Location) + assert.EqualValues(t, fake.Handler, getFaultRespBody.Handler) + }) +} + +func TestHandlers_makeSetFaultConfigHandler(t *testing.T) { + mockCtrl := gomock.NewController(t) + + t.Run("with empty name id specified", func(t *testing.T) { + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + server := newHttpTestServer(t, "/{name}", makeSetFaultConfigHandler(mockStore)) + defer server.Close() + + resp, _ := http.Get(fmt.Sprintf("%s/%s", server.URL, "")) + assert.NotNil(t, resp, "expected non-nil response") + assert.Equal(t, http.StatusNotFound, resp.StatusCode, "expected status codes to match") + }) + + t.Run("with name id and no fault", func(t *testing.T) { + fake := &fakeFault{ + Name: "fault", + Location: "location", + Handler: `{"name":"handler-name"}`, + } + + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + mockStore.EXPECT().Exists(fake.Name).Return(false) + + server := newHttpTestServer(t, "/{name}", makeSetFaultConfigHandler(mockStore)) + defer server.Close() + + url := fmt.Sprintf("%s/%s", server.URL, fake.Name) + request, err := http.NewRequest("PATCH", url, nil) + if err != nil { + assert.FailNow(t, err.Error()) + } + + resp, err := server.Client().Do(request) + if err != nil { + assert.FailNow(t, err.Error()) + } + assert.NotNil(t, resp, "expected non-nil response") + assert.Equal(t, http.StatusNotFound, resp.StatusCode, "expected status codes to match") + }) + + t.Run("with name id and valid handler config", func(t *testing.T) { + name := "fault-name" + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + mockStore.EXPECT().Exists(name).Return(true) + mockStore.EXPECT().Set(name, gomock.Any()).Return(nil) + + server := newHttpTestServer(t, "/{name}", makeSetFaultConfigHandler(mockStore)) + defer server.Close() + + // Simulate a PATCH request with a payload. + configRequest := `{"name": "pause", "duration": "400ms"}` + body, err := json.Marshal(configRequest) + if err != nil { + t.Fatalf("Failed to marshal JSON: %v", err) + } + + url := fmt.Sprintf("%s/%s", server.URL, name) + request, err := http.NewRequest("PATCH", url, bytes.NewBuffer(body)) + if err != nil { + assert.FailNow(t, err.Error()) + } + + // Invoke the API. + resp, err := server.Client().Do(request) + if err != nil { + assert.FailNow(t, err.Error()) + } + assert.NotNil(t, resp, "expected non-nil response") + assert.Equal(t, http.StatusAccepted, resp.StatusCode, "expected status codes to match") + }) + + t.Run("with name id and empty handler config", func(t *testing.T) { + name := "fault-name" + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + mockStore.EXPECT().Exists(name).Return(true) + + server := newHttpTestServer(t, "/{name}", makeSetFaultConfigHandler(mockStore)) + defer server.Close() + + // Simulate a PATCH request with a payload. + url := fmt.Sprintf("%s/%s", server.URL, name) + request, err := http.NewRequest("PATCH", url, bytes.NewBuffer(nil)) + if err != nil { + assert.FailNow(t, err.Error()) + } + + // Invoke the API. + resp, err := server.Client().Do(request) + if err != nil { + assert.FailNow(t, err.Error()) + } + assert.NotNil(t, resp, "expected non-nil response") + assert.Equal(t, http.StatusBadRequest, resp.StatusCode, "expected status codes to match") + }) + + t.Run("with name id and error when setting fault handler config", func(t *testing.T) { + name := "fault-name" + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + mockStore.EXPECT().Exists(name).Return(true) + mockStore.EXPECT().Set(name, gomock.Any()).Return(errors.New("failed to set handler config")) + + server := newHttpTestServer(t, "/{name}", makeSetFaultConfigHandler(mockStore)) + defer server.Close() + + // Simulate a PATCH request with a payload. + configRequest := `{"name": "pause", "duration": "400ms"}` + body, err := json.Marshal(configRequest) + if err != nil { + t.Fatalf("Failed to marshal JSON: %v", err) + } + + url := fmt.Sprintf("%s/%s", server.URL, name) + request, err := http.NewRequest("PATCH", url, bytes.NewBuffer(body)) + if err != nil { + assert.FailNow(t, err.Error()) + } + + // Invoke the API. + resp, err := server.Client().Do(request) + if err != nil { + assert.FailNow(t, err.Error()) + } + assert.NotNil(t, resp, "expected non-nil response") + assert.Equal(t, http.StatusInternalServerError, resp.StatusCode, "expected status codes to match") + }) +} + +func TestHandlers_makeResetFaultConfigHandler(t *testing.T) { + mockCtrl := gomock.NewController(t) + + tt := map[string]struct { + status int + name string + setMocks func(store *mock_rest.MockFaultStore) + assertNilOrNot assert.ValueAssertionFunc + }{ + "with name id but no fault found": { + status: http.StatusNotFound, + name: "name", + assertNilOrNot: assert.NotNil, + setMocks: func(store *mock_rest.MockFaultStore) { + store.EXPECT().Exists("name").Return(false) + }, + }, + "with name id and error during reset": { + status: http.StatusInternalServerError, + name: "name", + assertNilOrNot: assert.NotNil, + setMocks: func(store *mock_rest.MockFaultStore) { + store.EXPECT().Exists("name").Return(true) + store.EXPECT().Reset("name").Return(fmt.Errorf("reset failure")) + }, + }, + "with name id and no error during reset": { + name: "name", + status: http.StatusAccepted, + assertNilOrNot: assert.NotNil, + setMocks: func(store *mock_rest.MockFaultStore) { + store.EXPECT().Exists("name").Return(true) + store.EXPECT().Reset("name").Return(nil) + }, + }, + } + + for name, test := range tt { + t.Run(name, func(t *testing.T) { + mockStore := mock_rest.NewMockFaultStore(mockCtrl) + test.setMocks(mockStore) + + server := newHttpTestServer(t, "/{name}", makeResetFaultConfigHandler(mockStore)) + defer server.Close() + + request, err := http.NewRequest("DELETE", fmt.Sprintf("%s/%s", server.URL, test.name), nil) + if err != nil { + assert.FailNow(t, err.Error()) + } + resp, err := server.Client().Do(request) + if err != nil { + assert.FailNow(t, err.Error()) + } + + test.assertNilOrNot(t, resp) + assert.Equal(t, test.status, resp.StatusCode, "expected status codes to match") + }) + } +} diff --git a/internal/fiji/rest/rest.go b/internal/fiji/rest/rest.go new file mode 100644 index 000000000..106aa5d33 --- /dev/null +++ b/internal/fiji/rest/rest.go @@ -0,0 +1,21 @@ +package rest + +//go:generate mockgen -destination=../../../mocks/mock_internal/mock_fiji/mock_rest/mock_store.go github.com/netapp/trident/internal/fiji/rest FaultStore + +import ( + "github.com/netapp/trident/internal/fiji/store" +) + +type FaultStore interface { + Add(string, store.Fault) + Set(string, []byte) error + Reset(string) error + Get(string) (store.Fault, bool) + Exists(string) bool + List() []store.Fault +} + +// NewFaultInjectionServer is the main constructor for creating the FIJI API server plugin. +func NewFaultInjectionServer(address string, store FaultStore) *Server { + return NewHTTPServer(address, NewRouter(store)) +} diff --git a/internal/fiji/rest/router.go b/internal/fiji/rest/router.go new file mode 100644 index 000000000..e6b16729f --- /dev/null +++ b/internal/fiji/rest/router.go @@ -0,0 +1,21 @@ +package rest + +import ( + "net/http" + + "github.com/gorilla/mux" +) + +// NewRouter is used to set up HTTP and HTTPS endpoints for the controller +func NewRouter(store FaultStore) *mux.Router { + router := mux.NewRouter().StrictSlash(true) + for _, route := range makeRoutes(store) { + router. + Methods(route.Method). + Path(route.Pattern). + Name(route.Name). + Handler(http.Handler(route.HandlerFunc)) + } + + return router +} diff --git a/internal/fiji/rest/routes.go b/internal/fiji/rest/routes.go new file mode 100644 index 000000000..d51f45ece --- /dev/null +++ b/internal/fiji/rest/routes.go @@ -0,0 +1,49 @@ +package rest + +import ( + "net/http" +) + +const ( + apiPrefix = "/" + apiName + faultURL = apiPrefix + "/" + "fault" + faultByNameURL = faultURL + "/{name}" +) + +type Route struct { + Name string + Method string + Pattern string + HandlerFunc http.HandlerFunc +} + +type Routes []Route + +func makeRoutes(store FaultStore) Routes { + return Routes{ + Route{ + "ListFaultConfigs", + "GET", + faultURL, + makeListFaultsHandler(store), + }, + Route{ + "GetFaultConfig", + "GET", + faultByNameURL, + makeGetFaultHandler(store), + }, + Route{ + "SetFaultConfig", + "PATCH", + faultByNameURL, + makeSetFaultConfigHandler(store), + }, + Route{ + "DeleteFaultConfig", + "DELETE", + faultByNameURL, + makeResetFaultConfigHandler(store), + }, + } +} diff --git a/internal/fiji/rest/routes_test.go b/internal/fiji/rest/routes_test.go new file mode 100644 index 000000000..c71f0f8a6 --- /dev/null +++ b/internal/fiji/rest/routes_test.go @@ -0,0 +1,16 @@ +package rest + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRoutes_makeRoutes(t *testing.T) { + routes := makeRoutes(nil) + assert.NotEmpty(t, routes, "expected routes to be non-empty") + + for _, route := range routes { + assert.NotNil(t, route.HandlerFunc, "route handlers should never be nil") + } +} diff --git a/internal/fiji/rest/server.go b/internal/fiji/rest/server.go new file mode 100644 index 000000000..a6b46ea05 --- /dev/null +++ b/internal/fiji/rest/server.go @@ -0,0 +1,55 @@ +package rest + +import ( + "errors" + "net/http" + "time" + + . "github.com/netapp/trident/logging" +) + +const ( + apiName = "fiji" + apiVersion = "1" + httpTimeout = 90 * time.Second +) + +type Server struct { + server *http.Server +} + +func NewHTTPServer(address string, handler http.Handler) *Server { + return &Server{ + server: &http.Server{ + Addr: address, + Handler: handler, + ReadHeaderTimeout: httpTimeout, + }, + } +} + +func (s *Server) Activate() error { + go func() { + Log().WithField("address", s.server.Addr).Info("Activating FIJI API server.") + if err := s.server.ListenAndServe(); err != nil { + // If the server closes gracefully, log and exit the routine. + if errors.Is(err, http.ErrServerClosed) { + return + } + } + }() + return nil +} + +func (s *Server) Deactivate() error { + Log().WithField("address", s.server.Addr).Info("Deactivating FIJI API server.") + return s.server.Shutdown(nil) +} + +func (s *Server) GetName() string { + return apiName +} + +func (s *Server) Version() string { + return apiVersion +} diff --git a/internal/fiji/rest/server_test.go b/internal/fiji/rest/server_test.go new file mode 100644 index 000000000..d354c71c3 --- /dev/null +++ b/internal/fiji/rest/server_test.go @@ -0,0 +1,21 @@ +package rest + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func makeTestHandler() http.HandlerFunc { + return func(_ http.ResponseWriter, _ *http.Request) {} +} + +func TestServer_NewHTTPServer(t *testing.T) { + handler := makeTestHandler() + server := NewHTTPServer(":50000", handler) + assert.NoError(t, server.Activate(), "unexpected error") + assert.Equal(t, apiName, server.GetName(), "expected names to match") + assert.Equal(t, apiVersion, server.Version(), "expected versions to match") + assert.NoError(t, server.Deactivate(), "unexepcted error") +} diff --git a/internal/fiji/store/store.go b/internal/fiji/store/store.go new file mode 100644 index 000000000..d8a74e144 --- /dev/null +++ b/internal/fiji/store/store.go @@ -0,0 +1,112 @@ +package store + +//go:generate mockgen -destination=../../../mocks/mock_internal/mock_fiji/mock_store/mock_fault.go github.com/netapp/trident/internal/fiji/store Fault + +import ( + "fmt" + "sync" +) + +// Fault defines behaviors for faults stored in the store. +type Fault interface { + Inject() error + Reset() + IsHandlerSet() bool + SetHandler([]byte) error +} + +type Store struct { + mutex *sync.RWMutex + faults map[string]Fault +} + +func NewFaultStore() *Store { + return &Store{ + mutex: &sync.RWMutex{}, + faults: make(map[string]Fault), + } +} + +func (s *Store) Add(key string, fault Fault) { + s.mutex.Lock() + defer s.mutex.Unlock() + + if key == "" { + panic("empty fault name") + } else if fault == nil { + panic("nil fault") + } + + if _, ok := s.faults[key]; ok { + panic("duplicate fault name detected") + } + s.faults[key] = fault +} + +func (s *Store) Set(key string, config []byte) error { + s.mutex.Lock() + defer s.mutex.Unlock() + + if key == "" { + return fmt.Errorf("empty fault name") + } else if config == nil { + return fmt.Errorf("empty config") + } + + fault, ok := s.faults[key] + if !ok { + return fmt.Errorf("fault [%s] specified in config does not exist", key) + } + + // This will modify the fault state in-memory, so no need to reassign it at the end. + if err := fault.SetHandler(config); err != nil { + return fmt.Errorf("failed to load fault config for fault: [%s]; %v", key, err) + } + + return nil +} + +func (s *Store) Reset(key string) error { + s.mutex.Lock() + defer s.mutex.Unlock() + + if key == "" { + return fmt.Errorf("empty fault name") + } + + fault, ok := s.faults[key] + if !ok { + return fmt.Errorf("fault [%s] specified in config does not exist", key) + } + + fault.Reset() + return nil +} + +func (s *Store) Get(key string) (Fault, bool) { + s.mutex.RLock() + defer s.mutex.RUnlock() + + f, ok := s.faults[key] + return f, ok +} + +func (s *Store) Exists(key string) bool { + s.mutex.RLock() + defer s.mutex.RUnlock() + + _, ok := s.faults[key] + return ok +} + +func (s *Store) List() []Fault { + s.mutex.RLock() + defer s.mutex.RUnlock() + + faults := make([]Fault, 0) + for _, fault := range s.faults { + faults = append(faults, fault) + } + + return faults +} diff --git a/internal/fiji/store/store_test.go b/internal/fiji/store/store_test.go new file mode 100644 index 000000000..aa54cef77 --- /dev/null +++ b/internal/fiji/store/store_test.go @@ -0,0 +1,200 @@ +package store + +import ( + "errors" + "io" + "os" + "sync" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + + . "github.com/netapp/trident/logging" + "github.com/netapp/trident/mocks/mock_internal/mock_fiji/mock_store" +) + +func TestMain(m *testing.M) { + // Disable any standard log output + InitLogOutput(io.Discard) + os.Exit(m.Run()) +} + +func newEmptyTestStore() *Store { + return NewFaultStore() +} + +// newTestStore takes a list of functions that return a name and a fault and +// returns a filled fault store. +func newTestStore(fs ...func() (string, Fault)) *Store { + faults := make(map[string]Fault, len(fs)) + for _, f := range fs { + name, fault := f() + faults[name] = fault + } + + return &Store{ + mutex: &sync.RWMutex{}, + faults: faults, + } +} + +func TestStore_Add(t *testing.T) { + mockCtrl := gomock.NewController(t) + + tt := map[string]func(*testing.T){ + "with empty key": func(t *testing.T) { + store := newEmptyTestStore() + assert.Panics(t, func() { store.Add("", nil) }, "expected panic") + }, + "with nil fault": func(t *testing.T) { + store := newEmptyTestStore() + assert.Panics(t, func() { store.Add("validFaultName", nil) }, "expected panic") + }, + "with two faults with the same name": func(t *testing.T) { + name := "fault-A" + + // Create two different faults. + faultA := mock_store.NewMockFault(mockCtrl) + faultB := mock_store.NewMockFault(mockCtrl) + + store := newEmptyTestStore() + store.Add(name, faultA) // Register the first fault with the name. + + // Ensure that attempting to add a second fault with the same name induces a panic. + assert.Panics(t, func() { store.Add(name, faultB) }, "expected panic") + }, + "with two faults with different names": func(t *testing.T) { + nameA := "fault-A" + nameB := "fault-B" + + // Create two faults with different names. + faultA := mock_store.NewMockFault(mockCtrl) + faultB := mock_store.NewMockFault(mockCtrl) + + store := newEmptyTestStore() + store.Add(nameA, faultA) + assert.NotPanics(t, func() { store.Add(nameB, faultB) }, "unexpected panic") + }, + } + + for name, test := range tt { + t.Run(name, test) + } +} + +func TestStore_Set(t *testing.T) { + mockCtrl := gomock.NewController(t) + + tt := map[string]func(*testing.T){ + "with empty key": func(t *testing.T) { + store := newEmptyTestStore() + assert.Error(t, store.Set("", nil), "expected error") + }, + "with nil config": func(t *testing.T) { + store := newEmptyTestStore() + assert.Error(t, store.Set("fault-A", nil), "expected error") + }, + "when fault doesn't exist": func(t *testing.T) { + name := "fault-A" + store := newEmptyTestStore() + + // Ensure that attempting to add a second fault with the same name induces a panic. + assert.Error(t, store.Set(name, nil), "expected error") + }, + "when fault can't set handler config": func(t *testing.T) { + name := "fault-A" + config := []byte(`{"name":"panic" `) // invalid json + fault := mock_store.NewMockFault(mockCtrl) + fault.EXPECT().SetHandler(config).Return(errors.New("set handler config error")) + + store := newEmptyTestStore() + store.Add(name, fault) // Register the first fault with the name. + + assert.Error(t, store.Set(name, config), "expected error") + }, + "when fault config is set": func(t *testing.T) { + name := "fault-A" + config := []byte(`{"name":"panic"}`) + fault := mock_store.NewMockFault(mockCtrl) + fault.EXPECT().SetHandler(config).Return(nil) + + store := newEmptyTestStore() + store.Add(name, fault) + + assert.NoError(t, store.Set(name, config), "expected error") + }, + } + + for name, test := range tt { + t.Run(name, test) + } +} + +func TestStore_Reset(t *testing.T) { + mockCtrl := gomock.NewController(t) + + tt := map[string]func(*testing.T){ + "with empty key": func(t *testing.T) { + store := newEmptyTestStore() + assert.Error(t, store.Reset(""), "expected error") + }, + "when fault doesn't exist": func(t *testing.T) { + name := "fault-A" + store := newEmptyTestStore() + + // Ensure that attempting to add a second fault with the same name induces a panic. + assert.Error(t, store.Reset(name), "expected error") + }, + "when fault config is reset": func(t *testing.T) { + name := "fault-A" + fault := mock_store.NewMockFault(mockCtrl) + fault.EXPECT().Reset() + + store := newEmptyTestStore() + store.Add(name, fault) + + assert.NoError(t, store.Reset(name), "expected error") + + f, ok := store.Get(name) + if !ok { + t.Fatalf("failed to find test fault in store") + } + assert.NotNil(t, f) + }, + } + + for name, test := range tt { + t.Run(name, test) + } +} + +func TestStore_Get(t *testing.T) { + store := newEmptyTestStore() + fault, exists := store.Get("") + assert.Nil(t, fault, "expected nil fault") + assert.False(t, exists, "expected fault to not exist") +} + +func TestStore_Exists(t *testing.T) { + store := newEmptyTestStore() + exists := store.Exists("") + assert.False(t, exists, "expected fault to not exist") +} + +func TestStore_List(t *testing.T) { + mockCtrl := gomock.NewController(t) + faultA, faultB := "faultA", "faultB" + + store := newTestStore( + func() (string, Fault) { + return faultA, mock_store.NewMockFault(mockCtrl) + }, + func() (string, Fault) { + return faultB, mock_store.NewMockFault(mockCtrl) + }, + ) + list := store.List() + assert.NotEmpty(t, list, "expected non-empty list") + assert.Len(t, list, 2, "expected length to be 2") +} diff --git a/main.go b/main.go index 633fc0eb5..382b6f440 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,7 @@ import ( "github.com/netapp/trident/frontend/docker" "github.com/netapp/trident/frontend/metrics" "github.com/netapp/trident/frontend/rest" + "github.com/netapp/trident/internal/fiji" . "github.com/netapp/trident/logging" persistentstore "github.com/netapp/trident/persistent_store" "github.com/netapp/trident/utils" @@ -394,6 +395,8 @@ func main() { orchestrator.AddFrontend(ctx, dockerFrontend) preBootstrapFrontends = append(preBootstrapFrontends, dockerFrontend) + // Set up fault-injection server for docker (single instance should run). + preBootstrapFrontends = append(preBootstrapFrontends, fiji.NewFrontend(":50000")) } else if enableCSI { config.CurrentDriverContext = config.ContextCSI @@ -452,22 +455,29 @@ func main() { }).Info("Initializing CSI frontend.") var csiFrontend *csi.Plugin + var fijiFrontend frontend.Plugin switch *csiRole { case csi.CSIController: txnMonitor = true csiFrontend, err = csi.NewControllerPlugin(*csiNodeName, *csiEndpoint, *aesKey, orchestrator, &controllerHelper, *enableForceDetach) + + fijiFrontend = fiji.NewFrontend(":50001") case csi.CSINode: csiFrontend, err = csi.NewNodePlugin(*csiNodeName, *csiEndpoint, *httpsCACert, *httpsClientCert, *httpsClientKey, *aesKey, orchestrator, *csiUnsafeNodeDetach, &nodeHelper, *enableForceDetach, *iSCSISelfHealingInterval, *iSCSISelfHealingWaitTime, *nvmeSelfHealingInterval) enableMutualTLS = false handler = rest.NewNodeRouter(csiFrontend) + + fijiFrontend = fiji.NewFrontend(":50000") case csi.CSIAllInOne: txnMonitor = true csiFrontend, err = csi.NewAllInOnePlugin(*csiNodeName, *csiEndpoint, *httpsCACert, *httpsClientCert, *httpsClientKey, *aesKey, orchestrator, &controllerHelper, &nodeHelper, *csiUnsafeNodeDetach, *iSCSISelfHealingInterval, *iSCSISelfHealingWaitTime, *nvmeSelfHealingInterval) + + fijiFrontend = fiji.NewFrontend(":50000") } if err != nil { Log().Fatalf("Unable to start the CSI frontend. %v", err) @@ -483,6 +493,9 @@ func main() { orchestrator.AddFrontend(ctx, crdController) postBootstrapFrontends = append(postBootstrapFrontends, crdController) } + + // Add the FIJI frontend to the pre-bootstrap frontends. + preBootstrapFrontends = append(preBootstrapFrontends, fijiFrontend) } // Create HTTP REST frontend diff --git a/mocks/mock_internal/mock_fiji/mock_injector.go b/mocks/mock_internal/mock_fiji/mock_injector.go new file mode 100644 index 000000000..ffebb9fb8 --- /dev/null +++ b/mocks/mock_internal/mock_fiji/mock_injector.go @@ -0,0 +1,48 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/netapp/trident/internal/fiji (interfaces: Injector) + +// Package mock_fiji is a generated GoMock package. +package mock_fiji + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockInjector is a mock of Injector interface. +type MockInjector struct { + ctrl *gomock.Controller + recorder *MockInjectorMockRecorder +} + +// MockInjectorMockRecorder is the mock recorder for MockInjector. +type MockInjectorMockRecorder struct { + mock *MockInjector +} + +// NewMockInjector creates a new mock instance. +func NewMockInjector(ctrl *gomock.Controller) *MockInjector { + mock := &MockInjector{ctrl: ctrl} + mock.recorder = &MockInjectorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInjector) EXPECT() *MockInjectorMockRecorder { + return m.recorder +} + +// Inject mocks base method. +func (m *MockInjector) Inject() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Inject") + ret0, _ := ret[0].(error) + return ret0 +} + +// Inject indicates an expected call of Inject. +func (mr *MockInjectorMockRecorder) Inject() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inject", reflect.TypeOf((*MockInjector)(nil).Inject)) +} diff --git a/mocks/mock_internal/mock_fiji/mock_models/mock_handler.go b/mocks/mock_internal/mock_fiji/mock_models/mock_handler.go new file mode 100644 index 000000000..ec83d9095 --- /dev/null +++ b/mocks/mock_internal/mock_fiji/mock_models/mock_handler.go @@ -0,0 +1,48 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/netapp/trident/internal/fiji/models (interfaces: FaultHandler) + +// Package mock_models is a generated GoMock package. +package mock_models + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockFaultHandler is a mock of FaultHandler interface. +type MockFaultHandler struct { + ctrl *gomock.Controller + recorder *MockFaultHandlerMockRecorder +} + +// MockFaultHandlerMockRecorder is the mock recorder for MockFaultHandler. +type MockFaultHandlerMockRecorder struct { + mock *MockFaultHandler +} + +// NewMockFaultHandler creates a new mock instance. +func NewMockFaultHandler(ctrl *gomock.Controller) *MockFaultHandler { + mock := &MockFaultHandler{ctrl: ctrl} + mock.recorder = &MockFaultHandlerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFaultHandler) EXPECT() *MockFaultHandlerMockRecorder { + return m.recorder +} + +// Handle mocks base method. +func (m *MockFaultHandler) Handle() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Handle") + ret0, _ := ret[0].(error) + return ret0 +} + +// Handle indicates an expected call of Handle. +func (mr *MockFaultHandlerMockRecorder) Handle() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handle", reflect.TypeOf((*MockFaultHandler)(nil).Handle)) +} diff --git a/mocks/mock_internal/mock_fiji/mock_rest/mock_store.go b/mocks/mock_internal/mock_fiji/mock_rest/mock_store.go new file mode 100644 index 000000000..5007441d6 --- /dev/null +++ b/mocks/mock_internal/mock_fiji/mock_rest/mock_store.go @@ -0,0 +1,118 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/netapp/trident/internal/fiji/rest (interfaces: FaultStore) + +// Package mock_rest is a generated GoMock package. +package mock_rest + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + store "github.com/netapp/trident/internal/fiji/store" +) + +// MockFaultStore is a mock of FaultStore interface. +type MockFaultStore struct { + ctrl *gomock.Controller + recorder *MockFaultStoreMockRecorder +} + +// MockFaultStoreMockRecorder is the mock recorder for MockFaultStore. +type MockFaultStoreMockRecorder struct { + mock *MockFaultStore +} + +// NewMockFaultStore creates a new mock instance. +func NewMockFaultStore(ctrl *gomock.Controller) *MockFaultStore { + mock := &MockFaultStore{ctrl: ctrl} + mock.recorder = &MockFaultStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFaultStore) EXPECT() *MockFaultStoreMockRecorder { + return m.recorder +} + +// Add mocks base method. +func (m *MockFaultStore) Add(arg0 string, arg1 store.Fault) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Add", arg0, arg1) +} + +// Add indicates an expected call of Add. +func (mr *MockFaultStoreMockRecorder) Add(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockFaultStore)(nil).Add), arg0, arg1) +} + +// Exists mocks base method. +func (m *MockFaultStore) Exists(arg0 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// Exists indicates an expected call of Exists. +func (mr *MockFaultStoreMockRecorder) Exists(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockFaultStore)(nil).Exists), arg0) +} + +// Get mocks base method. +func (m *MockFaultStore) Get(arg0 string) (store.Fault, bool) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0) + ret0, _ := ret[0].(store.Fault) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockFaultStoreMockRecorder) Get(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockFaultStore)(nil).Get), arg0) +} + +// List mocks base method. +func (m *MockFaultStore) List() []store.Fault { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List") + ret0, _ := ret[0].([]store.Fault) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockFaultStoreMockRecorder) List() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockFaultStore)(nil).List)) +} + +// Reset mocks base method. +func (m *MockFaultStore) Reset(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Reset", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Reset indicates an expected call of Reset. +func (mr *MockFaultStoreMockRecorder) Reset(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reset", reflect.TypeOf((*MockFaultStore)(nil).Reset), arg0) +} + +// Set mocks base method. +func (m *MockFaultStore) Set(arg0 string, arg1 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Set", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Set indicates an expected call of Set. +func (mr *MockFaultStoreMockRecorder) Set(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockFaultStore)(nil).Set), arg0, arg1) +} diff --git a/mocks/mock_internal/mock_fiji/mock_store/mock_fault.go b/mocks/mock_internal/mock_fiji/mock_store/mock_fault.go new file mode 100644 index 000000000..f0c7429bc --- /dev/null +++ b/mocks/mock_internal/mock_fiji/mock_store/mock_fault.go @@ -0,0 +1,88 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/netapp/trident/internal/fiji/store (interfaces: Fault) + +// Package mock_store is a generated GoMock package. +package mock_store + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockFault is a mock of Fault interface. +type MockFault struct { + ctrl *gomock.Controller + recorder *MockFaultMockRecorder +} + +// MockFaultMockRecorder is the mock recorder for MockFault. +type MockFaultMockRecorder struct { + mock *MockFault +} + +// NewMockFault creates a new mock instance. +func NewMockFault(ctrl *gomock.Controller) *MockFault { + mock := &MockFault{ctrl: ctrl} + mock.recorder = &MockFaultMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFault) EXPECT() *MockFaultMockRecorder { + return m.recorder +} + +// Inject mocks base method. +func (m *MockFault) Inject() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Inject") + ret0, _ := ret[0].(error) + return ret0 +} + +// Inject indicates an expected call of Inject. +func (mr *MockFaultMockRecorder) Inject() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inject", reflect.TypeOf((*MockFault)(nil).Inject)) +} + +// IsHandlerSet mocks base method. +func (m *MockFault) IsHandlerSet() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsHandlerSet") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsHandlerSet indicates an expected call of IsHandlerSet. +func (mr *MockFaultMockRecorder) IsHandlerSet() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsHandlerSet", reflect.TypeOf((*MockFault)(nil).IsHandlerSet)) +} + +// Reset mocks base method. +func (m *MockFault) Reset() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Reset") +} + +// Reset indicates an expected call of Reset. +func (mr *MockFaultMockRecorder) Reset() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reset", reflect.TypeOf((*MockFault)(nil).Reset)) +} + +// SetHandler mocks base method. +func (m *MockFault) SetHandler(arg0 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetHandler", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetHandler indicates an expected call of SetHandler. +func (mr *MockFaultMockRecorder) SetHandler(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetHandler", reflect.TypeOf((*MockFault)(nil).SetHandler), arg0) +}