[release-0.20] Shuffle hardware inventory for tinkerbell before reser…

…vation (#8898)

* shuffle hardware inventory for tinkerbell before reservation

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>

* Account for mutexes during shuffle and move it to struct method

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>

---------

Signed-off-by: Rahul Ganesh <rahulgab@amazon.com>
Co-authored-by: Rahul Ganesh <rahulgab@amazon.com>

internal/test/e2e/run.go

@@ -219,6 +219,13 @@ func RunTests(conf instanceRunConf, inventoryCatalogue map[string]*hardwareCatal
 		} else {
 			hardwareCatalogue = inventoryCatalogue[nonAirgappedHardware]
 		}
+		conf.Logger.Info("Shuffling hardware inventory for tinkerbell")
+		// shuffle hardware to introduce randomness during hardware reservation.
+		// we do not want quick e2e runs to always pick the first few available hardware from the list and over-populate the boot entries
+		// this will quickly break the booting process as the hardware runs out of boot space to store these entries.
+		// randomly picking the hardware will distribute the boot entries across these hardware during each run
+		// ideally for long term we want a clear cleanup of the boot entries in the hardware
+		hardwareCatalogue.shuffleHardware()
 		err = reserveTinkerbellHardware(&conf, hardwareCatalogue)
 		if err != nil {
 			return "", nil, err

internal/test/e2e/tinkerbell_hardware_queue.go

@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"fmt"
+	"math/rand"
 	"sync"
 	"time"
 
@@ -42,6 +43,15 @@ func (hwQu *hardwareCatalogue) releaseHardware(hws []*api.Hardware) {
 	hwQu.mu.Unlock()
 }
 
+func (hwQu *hardwareCatalogue) shuffleHardware() {
+	hwQu.mu.Lock()
+	random := rand.New(rand.NewSource(time.Now().UnixNano()))
+	random.Shuffle(len(hwQu.hws), func(i, j int) {
+		hwQu.hws[i], hwQu.hws[j] = hwQu.hws[j], hwQu.hws[i]
+	})
+	hwQu.mu.Unlock()
+}
+
 func newHardwareCatalogue(hws []*api.Hardware) *hardwareCatalogue {
 	return &hardwareCatalogue{
 		hws: hws,