From c563ace7509dd1057c38d2e09aae88e4715964ae Mon Sep 17 00:00:00 2001 From: Mihaela Balutoiu Date: Mon, 6 Nov 2023 12:19:41 +0200 Subject: [PATCH] Add integration tests for `test external provider` Signed-off-by: Mihaela Balutoiu --- test/integration/config/config.toml | 8 +++ test/integration/e2e/client_utils.go | 4 +- test/integration/e2e/e2e.go | 2 +- test/integration/e2e/instances.go | 25 ++++++--- test/integration/e2e/jobs.go | 4 +- test/integration/e2e/organizations.go | 3 +- test/integration/e2e/repositories.go | 12 +++- test/integration/main.go | 25 +++++++++ .../provider/garm-external-provider | 56 +++++++++++++++++++ test/integration/scripts/setup-garm.sh | 5 ++ 10 files changed, 128 insertions(+), 16 deletions(-) create mode 100755 test/integration/provider/garm-external-provider diff --git a/test/integration/config/config.toml b/test/integration/config/config.toml index 8fbd139c..1a67a409 100644 --- a/test/integration/config/config.toml +++ b/test/integration/config/config.toml @@ -50,6 +50,14 @@ description = "Local LXD installation" protocol = "simplestreams" skip_verify = false +[[provider]] +name = "test_external" +description = "external test provider" +provider_type = "external" + [provider.external] + config_file = "/etc/garm/test-provider/config" + provider_executable = "/etc/garm/test-provider/garm-external-provider" + [[github]] name = "${CREDENTIALS_NAME}" description = "GARM GitHub OAuth token" diff --git a/test/integration/e2e/client_utils.go b/test/integration/e2e/client_utils.go index 1ae2ecff..955f7b52 100644 --- a/test/integration/e2e/client_utils.go +++ b/test/integration/e2e/client_utils.go @@ -387,9 +387,9 @@ func getInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWrit return &getInstancesResponse.Payload, nil } -func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string) error { +func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string, forceRemove bool) error { return apiCli.Instances.DeleteInstance( - clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID), + clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID).WithForceRemove(&forceRemove), apiAuthToken) } diff --git a/test/integration/e2e/e2e.go b/test/integration/e2e/e2e.go index 63cfbc5a..b94c0652 100644 --- a/test/integration/e2e/e2e.go +++ b/test/integration/e2e/e2e.go @@ -72,7 +72,7 @@ func GracefulCleanup() { panic(err) } for _, instance := range poolInstances { - if err := deleteInstance(cli, authToken, instance.Name); err != nil { + if err := deleteInstance(cli, authToken, instance.Name, false); err != nil { panic(err) } log.Printf("Instance %s deletion initiated", instance.Name) diff --git a/test/integration/e2e/instances.go b/test/integration/e2e/instances.go index db10c71b..7b0de20b 100644 --- a/test/integration/e2e/instances.go +++ b/test/integration/e2e/instances.go @@ -33,7 +33,14 @@ func waitInstanceStatus(name string, status commonParams.InstanceStatus, runnerS return nil, fmt.Errorf("timeout waiting for instance %s status to reach status %s and runner status %s", name, status, runnerStatus) } -func waitInstanceToBeRemoved(name string, timeout time.Duration) error { +func DeleteInstance(name string, forceRemove bool) { + if err := deleteInstance(cli, authToken, name, forceRemove); err != nil { + panic(err) + } + log.Printf("Instance %s deletion initiated", name) +} + +func WaitInstanceToBeRemoved(name string, timeout time.Duration) error { var timeWaited time.Duration = 0 var instance *params.Instance @@ -67,7 +74,7 @@ func waitInstanceToBeRemoved(name string, timeout time.Duration) error { return fmt.Errorf("instance %s was not removed within the timeout", name) } -func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error { +func WaitPoolInstances(poolID string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) error { var timeWaited time.Duration = 0 pool, err := getPool(cli, authToken, poolID) @@ -75,22 +82,22 @@ func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error { return err } - log.Printf("Waiting for pool %s to have all instances as idle running", poolID) + log.Printf("Waiting for pool %s instances to reach status: %s and runner status: %s", poolID, status, runnerStatus) for timeWaited < timeout { poolInstances, err := listPoolInstances(cli, authToken, poolID) if err != nil { return err } - runningIdleCount := 0 + instancesCount := 0 for _, instance := range poolInstances { - if instance.Status == commonParams.InstanceRunning && instance.RunnerStatus == params.RunnerIdle { - runningIdleCount++ + if instance.Status == status && instance.RunnerStatus == runnerStatus { + instancesCount++ } } - log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount) - if runningIdleCount == int(pool.MinIdleRunners) && runningIdleCount == len(poolInstances) { + log.Printf("Pool %s instance reached status: %s and runner status: %s: %d/%d", poolID, status, runnerStatus, instancesCount, len(poolInstances)) + if instancesCount == int(pool.MinIdleRunners) && instancesCount == len(poolInstances) { return nil } time.Sleep(5 * time.Second) @@ -99,5 +106,5 @@ func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error { _ = dumpPoolInstancesDetails(pool.ID) - return fmt.Errorf("timeout waiting for pool %s to have all idle instances running", poolID) + return fmt.Errorf("timeout waiting for pool %s instances to reach status: %s and runner status: %s", poolID, status, runnerStatus) } diff --git a/test/integration/e2e/jobs.go b/test/integration/e2e/jobs.go index 7a1c0f9d..d8abccf2 100644 --- a/test/integration/e2e/jobs.go +++ b/test/integration/e2e/jobs.go @@ -41,13 +41,13 @@ func ValidateJobLifecycle(label string) { } // wait for instance to be removed - err = waitInstanceToBeRemoved(instance.Name, 5*time.Minute) + err = WaitInstanceToBeRemoved(instance.Name, 5*time.Minute) if err != nil { panic(err) } // wait for GARM to rebuild the pool running idle instances - err = waitPoolRunningIdleInstances(instance.PoolID, 6*time.Minute) + err = WaitPoolInstances(instance.PoolID, commonParams.InstanceRunning, params.RunnerIdle, 5*time.Minute) if err != nil { panic(err) } diff --git a/test/integration/e2e/organizations.go b/test/integration/e2e/organizations.go index 277d36cc..490ca784 100644 --- a/test/integration/e2e/organizations.go +++ b/test/integration/e2e/organizations.go @@ -4,6 +4,7 @@ import ( "log" "time" + commonParams "github.com/cloudbase/garm-provider-common/params" "github.com/cloudbase/garm/params" ) @@ -100,7 +101,7 @@ func WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) { panic(err) } for _, pool := range orgPools { - err := waitPoolRunningIdleInstances(pool.ID, timeout) + err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout) if err != nil { _ = dumpOrgInstancesDetails(orgID) panic(err) diff --git a/test/integration/e2e/repositories.go b/test/integration/e2e/repositories.go index 2810393b..72c10e32 100644 --- a/test/integration/e2e/repositories.go +++ b/test/integration/e2e/repositories.go @@ -4,6 +4,7 @@ import ( "log" "time" + commonParams "github.com/cloudbase/garm-provider-common/params" "github.com/cloudbase/garm/params" ) @@ -95,13 +96,22 @@ func DeleteRepoPool(repoID, repoPoolID string) { } } +func DisableRepoPool(repoID, repoPoolID string) { + log.Printf("Disable repo %s pool %s", repoID, repoPoolID) + enabled := false + poolParams := params.UpdatePoolParams{Enabled: &enabled} + if _, err := updateRepoPool(cli, authToken, repoID, repoPoolID, poolParams); err != nil { + panic(err) + } +} + func WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) { repoPools, err := listRepoPools(cli, authToken, repoID) if err != nil { panic(err) } for _, pool := range repoPools { - err := waitPoolRunningIdleInstances(pool.ID, timeout) + err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout) if err != nil { _ = dumpRepoInstancesDetails(repoID) panic(err) diff --git a/test/integration/main.go b/test/integration/main.go index 0918d964..44157539 100644 --- a/test/integration/main.go +++ b/test/integration/main.go @@ -34,6 +34,17 @@ var ( Tags: []string{"repo-runner"}, Enabled: true, } + repoPoolParams2 = params.CreatePoolParams{ + MaxRunners: 2, + MinIdleRunners: 0, + Flavor: "default", + Image: "ubuntu:22.04", + OSType: commonParams.Linux, + OSArch: commonParams.Amd64, + ProviderName: "test_external", + Tags: []string{"repo-runner-2"}, + Enabled: true, + } orgName = os.Getenv("ORG_NAME") orgWebhookSecret = os.Getenv("ORG_WEBHOOK_SECRET") @@ -101,6 +112,20 @@ func main() { repoPool = e2e.CreateRepoPool(repo.ID, repoPoolParams) _ = e2e.UpdateRepoPool(repo.ID, repoPool.ID, repoPoolParams.MaxRunners, 1) + ///////////////////////////// + // Test external provider /// + ///////////////////////////// + repoPool2 := e2e.CreateRepoPool(repo.ID, repoPoolParams2) + _ = e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1) + e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstanceRunning, params.RunnerPending, 1*time.Minute) + repoPool2 = e2e.GetRepoPool(repo.ID, repoPool2.ID) + e2e.DisableRepoPool(repo.ID, repoPool2.ID) + e2e.DeleteInstance(repoPool2.Instances[0].Name, false) + e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstancePendingDelete, params.RunnerPending, 1*time.Minute) + e2e.DeleteInstance(repoPool2.Instances[0].Name, true) // delete instance with forceRemove + e2e.WaitInstanceToBeRemoved(repoPool2.Instances[0].Name, 1*time.Minute) + e2e.DeleteRepoPool(repo.ID, repoPool2.ID) + /////////////////// // organizations // /////////////////// diff --git a/test/integration/provider/garm-external-provider b/test/integration/provider/garm-external-provider new file mode 100755 index 00000000..88e6f46e --- /dev/null +++ b/test/integration/provider/garm-external-provider @@ -0,0 +1,56 @@ +#!/bin/bash + +set -e +set -o pipefail + +if [ ! -t 0 ] +then + INPUT=$(cat -) +fi + +if [ -z "$GARM_PROVIDER_CONFIG_FILE" ] +then + echo "no config file specified in env" + exit 1 +fi + +source "$GARM_PROVIDER_CONFIG_FILE" + +function CreateInstance() { + if [ -z "$INPUT" ]; then + echo "expected build params in stdin" + exit 1 + fi + + jq -rnc '{"provider_id": "test-provider-id", "name": "test-instance-name", "os_type": "linux", "os_name": "ubuntu", "os_version": "20.04", "os_arch": "x86_64", "status": "running"}' +} + +case "$GARM_COMMAND" in + "CreateInstance") + CreateInstance + ;; + "DeleteInstance") + echo "RemoveAllInstances not implemented" + exit 1 + ;; + "GetInstance") + echo "Get instance with id: ${GARM_INSTANCE_ID}" + ;; + "ListInstances") + echo "List instances with pool id: ${GARM_POOL_ID}" + ;; + "StartInstance") + echo "Start instance: ${GARM_INSTANCE_NAME} with id: ${GARM_INSTANCE_ID}" + ;; + "StopInstance") + echo "Stop instance: ${GARM_INSTANCE_NAME} with id: ${GARM_INSTANCE_ID}" + ;; + "RemoveAllInstances") + echo "RemoveAllInstances not implemented" + exit 1 + ;; + *) + echo "Invalid GARM provider command: \"$GARM_COMMAND\"" + exit 1 + ;; +esac diff --git a/test/integration/scripts/setup-garm.sh b/test/integration/scripts/setup-garm.sh index 525c2f92..91dddddf 100755 --- a/test/integration/scripts/setup-garm.sh +++ b/test/integration/scripts/setup-garm.sh @@ -5,6 +5,7 @@ DIR="$(dirname $0)" BINARIES_DIR="$PWD/bin" CONTRIB_DIR="$PWD/contrib" CONFIG_DIR="$PWD/test/integration/config" +CONFIG_DIR_PROV="$PWD/test/integration/provider" if [[ ! -f $BINARIES_DIR/garm ]] || [[ ! -f $BINARIES_DIR/garm-cli ]]; then echo "ERROR: Please build GARM binaries first" @@ -46,6 +47,10 @@ sudo mkdir -p /etc/garm cat $CONFIG_DIR/config.toml | envsubst | sudo tee /etc/garm/config.toml sudo chown -R garm:garm /etc/garm +sudo mkdir /etc/garm/test-provider +sudo touch $CONFIG_DIR_PROV/config +sudo cp $CONFIG_DIR_PROV/* /etc/garm/test-provider + sudo mv $BINARIES_DIR/* /usr/local/bin/ sudo cp $CONTRIB_DIR/garm.service /etc/systemd/system/garm.service