Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove self-hosted runners in ci_workflow_gen #15989

Merged
merged 2 commits into from
May 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 0 additions & 91 deletions GITHUB_SELF_HOSTED_RUNNERS.md

This file was deleted.

138 changes: 1 addition & 137 deletions test/ci_workflow_gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,7 @@ const (
// to be used.
clusterTestTemplate = "templates/cluster_endtoend_test%s.tpl"

unitTestSelfHostedTemplate = "templates/unit_test_self_hosted.tpl"
unitTestSelfHostedDatabases = ""
dockerFileTemplate = "templates/dockerfile.tpl"
clusterTestSelfHostedTemplate = "templates/cluster_endtoend_test_self_hosted.tpl"
clusterTestDockerTemplate = "templates/cluster_endtoend_test_docker.tpl"
clusterTestDockerTemplate = "templates/cluster_endtoend_test_docker.tpl"
)

var (
Expand Down Expand Up @@ -126,7 +122,6 @@ var (
"vttablet_prscomplex",
}

clusterSelfHostedList = []string{}
clusterDockerList = []string{}
clustersRequiringXtraBackup = []string{
"xb_backup",
Expand Down Expand Up @@ -168,12 +163,6 @@ type clusterTest struct {
Cores16 bool
}

type selfHostedTest struct {
Name, Platform, Dockerfile, Shard, ImageName, directoryName string
FileName string
MakeTools, InstallXtraBackup, Docker bool
}

// clusterMySQLVersions return list of mysql versions (one or more) that this cluster needs to test against
func clusterMySQLVersions(clusterName string) mysqlVersions {
switch {
Expand Down Expand Up @@ -211,16 +200,6 @@ func main() {
generateUnitTestWorkflows()
generateClusterWorkflows(clusterList, clusterTestTemplate)
generateClusterWorkflows(clusterDockerList, clusterTestDockerTemplate)

// tests that will use self-hosted runners
err := generateSelfHostedUnitTestWorkflows()
if err != nil {
log.Fatal(err)
}
err = generateSelfHostedClusterWorkflows()
if err != nil {
log.Fatal(err)
}
}

func canonnizeList(list []string) []string {
Expand All @@ -233,98 +212,6 @@ func canonnizeList(list []string) []string {
return output
}

func parseList(csvList string) []string {
var list []string
for _, item := range strings.Split(csvList, ",") {
if item != "" {
list = append(list, strings.TrimSpace(item))
}
}
return list
}

func generateSelfHostedUnitTestWorkflows() error {
platforms := parseList(unitTestSelfHostedDatabases)
for _, platform := range platforms {
directoryName := fmt.Sprintf("unit_test_%s", platform)
test := &selfHostedTest{
Name: fmt.Sprintf("Unit Test (%s)", platform),
ImageName: fmt.Sprintf("unit_test_%s", platform),
Platform: platform,
directoryName: directoryName,
Dockerfile: fmt.Sprintf("./.github/docker/%s/Dockerfile", directoryName),
MakeTools: true,
InstallXtraBackup: false,
}
err := setupTestDockerFile(test)
if err != nil {
return err
}
test.FileName = fmt.Sprintf("unit_test_%s.yml", platform)
filePath := fmt.Sprintf("%s/%s", workflowConfigDir, test.FileName)
err = writeFileFromTemplate(unitTestSelfHostedTemplate, filePath, test)
if err != nil {
log.Print(err)
}
}
return nil
}

func generateSelfHostedClusterWorkflows() error {
clusters := canonnizeList(clusterSelfHostedList)
for _, cluster := range clusters {
for _, mysqlVersion := range clusterMySQLVersions(cluster) {
// check mysqlversion
mysqlVersionIndicator := ""
if mysqlVersion != defaultMySQLVersion && len(clusterMySQLVersions(cluster)) > 1 {
mysqlVersionIndicator = "_" + string(mysqlVersion)
}

directoryName := fmt.Sprintf("cluster_test_%s%s", cluster, mysqlVersionIndicator)
test := &selfHostedTest{
Name: fmt.Sprintf("Cluster (%s)(%s)", cluster, mysqlVersion),
ImageName: fmt.Sprintf("cluster_test_%s%s", cluster, mysqlVersionIndicator),
Platform: "mysql80",
directoryName: directoryName,
Dockerfile: fmt.Sprintf("./.github/docker/%s/Dockerfile", directoryName),
Shard: cluster,
MakeTools: false,
InstallXtraBackup: false,
}
makeToolClusters := canonnizeList(clustersRequiringMakeTools)
for _, makeToolCluster := range makeToolClusters {
if makeToolCluster == cluster {
test.MakeTools = true
break
}
}
xtraBackupClusters := canonnizeList(clustersRequiringXtraBackup)
for _, xtraBackupCluster := range xtraBackupClusters {
if xtraBackupCluster == cluster {
test.InstallXtraBackup = true
break
}
}
if mysqlVersion == mysql57 {
test.Platform = string(mysql57)
}

err := setupTestDockerFile(test)
if err != nil {
return err
}

test.FileName = fmt.Sprintf("cluster_endtoend_%s%s.yml", cluster, mysqlVersionIndicator)
filePath := fmt.Sprintf("%s/%s", workflowConfigDir, test.FileName)
err = writeFileFromTemplate(clusterTestSelfHostedTemplate, filePath, test)
if err != nil {
log.Print(err)
}
}
}
return nil
}

func generateClusterWorkflows(list []string, tpl string) {
clusters := canonnizeList(list)
for _, cluster := range clusters {
Expand Down Expand Up @@ -420,29 +307,6 @@ func evalengineToString(evalengine string) string {
return ""
}

func setupTestDockerFile(test *selfHostedTest) error {
// remove the directory
relDirectoryName := fmt.Sprintf("../.github/docker/%s", test.directoryName)
err := os.RemoveAll(relDirectoryName)
if err != nil {
return err
}
// create the directory
err = os.MkdirAll(relDirectoryName, 0755)
if err != nil {
return err
}

// generate the docker file
dockerFilePath := path.Join(relDirectoryName, "Dockerfile")
err = writeFileFromTemplate(dockerFileTemplate, dockerFilePath, test)
if err != nil {
return err
}

return nil
}

func writeFileFromTemplate(templateFile, filePath string, test any) error {
tpl := template.New(path.Base(templateFile))
tpl.Funcs(template.FuncMap{
Expand Down
91 changes: 0 additions & 91 deletions test/templates/cluster_endtoend_test_self_hosted.tpl

This file was deleted.

Loading
Loading