From 926ed345ea77b859fe7a642f8053a9dabef00049 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 19 Sep 2024 12:18:51 +0200 Subject: [PATCH] Added git lfs pull to clone of pipeline (#1191) * Added git lfs pull to clone of pipeline * Fix * Fixed unit-tests * Reworked clone volume mounts * Cleanup * Cleanup * Cleanup --- charts/radix-operator/Chart.yaml | 2 +- .../internal/jobs/build/acr_test.go | 4 +- .../internal/jobs/build/buildkit_test.go | 4 +- pipeline-runner/internal/jobs/build/common.go | 8 +++ pipeline-runner/internal/tekton/tekton.go | 3 ++ pipeline-runner/steps/preparepipeline/step.go | 2 +- pkg/apis/defaults/environment_variables.go | 4 +- pkg/apis/utils/git/clone.go | 50 +++++++++---------- pkg/apis/utils/git/clone_test.go | 2 +- pkg/apis/utils/git/defaults.go | 27 ++++++++++ 10 files changed, 73 insertions(+), 33 deletions(-) create mode 100644 pkg/apis/utils/git/defaults.go diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml index f0a6591f4..96d3e00e0 100644 --- a/charts/radix-operator/Chart.yaml +++ b/charts/radix-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: radix-operator version: 1.39.0 -appVersion: 1.59.0 +appVersion: 1.59.1 kubeVersion: ">=1.24.0" description: Radix Operator keywords: diff --git a/pipeline-runner/internal/jobs/build/acr_test.go b/pipeline-runner/internal/jobs/build/acr_test.go index 28ee4ce01..d998c66f2 100644 --- a/pipeline-runner/internal/jobs/build/acr_test.go +++ b/pipeline-runner/internal/jobs/build/acr_test.go @@ -106,6 +106,7 @@ func assertACRJobSpec(t *testing.T, pushImage bool) { {Name: git.GitSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: git.GitSSHKeyVolumeName, DefaultMode: pointers.Ptr[int32](256)}}}, {Name: defaults.AzureACRServicePrincipleSecretName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: defaults.AzureACRServicePrincipleSecretName}}}, {Name: "radix-image-builder-home", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(5, resource.Mega)}}}, + {Name: git.CloneRepoHomeVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(5, resource.Mega)}}}, } for _, image := range componentImages { expectedVolumes = append(expectedVolumes, @@ -119,11 +120,12 @@ func assertACRJobSpec(t *testing.T, pushImage bool) { assert.ElementsMatch(t, []string{"internal-nslookup", "clone", "internal-chmod"}, slice.Map(job.Spec.Template.Spec.InitContainers, func(c corev1.Container) string { return c.Name })) cloneContainer, _ := slice.FindFirst(job.Spec.Template.Spec.InitContainers, func(c corev1.Container) bool { return c.Name == "clone" }) assert.Equal(t, args.GitCloneGitImage, cloneContainer.Image) - assert.Equal(t, []string{"git", "clone", "--recurse-submodules", cloneURL, "-b", args.Branch, "--verbose", "--progress", git.Workspace}, cloneContainer.Command) + assert.Equal(t, []string{"sh", "-c", "git config --global --add safe.directory /workspace && git clone --recurse-submodules anycloneurl -b anybranch --verbose --progress /workspace && cd /workspace && if [ -n \"$(git lfs ls-files 2>/dev/null)\" ]; then git lfs install && echo 'Pulling large files...' && git lfs pull && echo 'Done'; fi && cd -"}, cloneContainer.Command) assert.Empty(t, cloneContainer.Args) expectedCloneVolumeMounts := []corev1.VolumeMount{ {Name: git.BuildContextVolumeName, MountPath: git.Workspace}, {Name: git.GitSSHKeyVolumeName, MountPath: "/.ssh", ReadOnly: true}, + {Name: git.CloneRepoHomeVolumeName, MountPath: git.CloneRepoHomeVolumePath}, } assert.ElementsMatch(t, expectedCloneVolumeMounts, cloneContainer.VolumeMounts) diff --git a/pipeline-runner/internal/jobs/build/buildkit_test.go b/pipeline-runner/internal/jobs/build/buildkit_test.go index f62fc0e65..260938751 100644 --- a/pipeline-runner/internal/jobs/build/buildkit_test.go +++ b/pipeline-runner/internal/jobs/build/buildkit_test.go @@ -133,6 +133,7 @@ func assertBuildKitJobSpec(t *testing.T, useCache, pushImage bool, buildSecrets {Name: "radix-image-builder-home", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(5, resource.Mega)}}}, {Name: "build-kit-run", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(100, resource.Giga)}}}, {Name: "build-kit-root", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(100, resource.Giga)}}}, + {Name: git.CloneRepoHomeVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: resource.NewScaledQuantity(5, resource.Mega)}}}, } if len(args.ExternalContainerRegistryDefaultAuthSecret) > 0 { expectedVolumes = append(expectedVolumes, corev1.Volume{ @@ -150,11 +151,12 @@ func assertBuildKitJobSpec(t *testing.T, useCache, pushImage bool, buildSecrets assert.ElementsMatch(t, []string{"internal-nslookup", "clone", "internal-chmod"}, slice.Map(job.Spec.Template.Spec.InitContainers, func(c corev1.Container) string { return c.Name })) cloneContainer, _ := slice.FindFirst(job.Spec.Template.Spec.InitContainers, func(c corev1.Container) bool { return c.Name == "clone" }) assert.Equal(t, args.GitCloneGitImage, cloneContainer.Image) - assert.Equal(t, []string{"git", "clone", "--recurse-submodules", cloneURL, "-b", args.Branch, "--verbose", "--progress", git.Workspace}, cloneContainer.Command) + assert.Equal(t, []string{"sh", "-c", "git config --global --add safe.directory /workspace && git clone --recurse-submodules anycloneurl -b anybranch --verbose --progress /workspace && cd /workspace && if [ -n \"$(git lfs ls-files 2>/dev/null)\" ]; then git lfs install && echo 'Pulling large files...' && git lfs pull && echo 'Done'; fi && cd -"}, cloneContainer.Command) assert.Empty(t, cloneContainer.Args) expectedCloneVolumeMounts := []corev1.VolumeMount{ {Name: git.BuildContextVolumeName, MountPath: git.Workspace}, {Name: git.GitSSHKeyVolumeName, MountPath: "/.ssh", ReadOnly: true}, + {Name: git.CloneRepoHomeVolumeName, MountPath: git.CloneRepoHomeVolumePath}, } assert.ElementsMatch(t, expectedCloneVolumeMounts, cloneContainer.VolumeMounts) diff --git a/pipeline-runner/internal/jobs/build/common.go b/pipeline-runner/internal/jobs/build/common.go index 58aae63c8..97c7e5b2d 100644 --- a/pipeline-runner/internal/jobs/build/common.go +++ b/pipeline-runner/internal/jobs/build/common.go @@ -78,6 +78,14 @@ func getCommonPodVolumes(componentImages []pipeline.BuildComponentImage) []corev }, }, }, + { + Name: git.CloneRepoHomeVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + SizeLimit: resource.NewScaledQuantity(5, resource.Mega), + }, + }, + }, } for _, image := range componentImages { diff --git a/pipeline-runner/internal/tekton/tekton.go b/pipeline-runner/internal/tekton/tekton.go index 7f7c38daf..5c094208b 100644 --- a/pipeline-runner/internal/tekton/tekton.go +++ b/pipeline-runner/internal/tekton/tekton.go @@ -97,6 +97,9 @@ func getJobVolumes() []corev1.Volume { { Name: git.BuildContextVolumeName, }, + { + Name: git.CloneRepoHomeVolumeName, + }, { Name: git.GitSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{ diff --git a/pipeline-runner/steps/preparepipeline/step.go b/pipeline-runner/steps/preparepipeline/step.go index 06430092e..d538dcd99 100644 --- a/pipeline-runner/steps/preparepipeline/step.go +++ b/pipeline-runner/steps/preparepipeline/step.go @@ -200,7 +200,7 @@ func (cli *PreparePipelinesStepImplementation) getPreparePipelinesJobConfig(pipe Value: strings.Join(pipelineInfo.PipelineArguments.DNSConfig.ReservedDNSAliases, ","), }, } - initContainers := git.CloneInitContainersWithContainerName(registration.Spec.CloneURL, configBranch, git.CloneConfigContainerName, internalgit.CloneConfigFromPipelineArgs(pipelineInfo.PipelineArguments)) + initContainers := git.CloneInitContainersWithContainerName(registration.Spec.CloneURL, configBranch, git.CloneConfigContainerName, internalgit.CloneConfigFromPipelineArgs(pipelineInfo.PipelineArguments), false) return internaltekton.CreateActionPipelineJob(defaults.RadixPipelineJobPreparePipelinesContainerName, action, pipelineInfo, appName, initContainers, &envVars) } diff --git a/pkg/apis/defaults/environment_variables.go b/pkg/apis/defaults/environment_variables.go index 12ed39158..87da14673 100644 --- a/pkg/apis/defaults/environment_variables.go +++ b/pkg/apis/defaults/environment_variables.go @@ -223,10 +223,10 @@ const ( // RadixGitCloneGitImageEnvironmentVariable The container image containing git, used in pipeline git clone init containers RadixGitCloneGitImageEnvironmentVariable = "RADIX_PIPELINE_GIT_CLONE_GIT_IMAGE" - // RadixGitCloneGitImageEnvironmentVariable The container image containing bash, used in pipeline git clone init containers + // RadixGitCloneBashImageEnvironmentVariable The container image containing bash, used in pipeline git clone init containers RadixGitCloneBashImageEnvironmentVariable = "RADIX_PIPELINE_GIT_CLONE_BASH_IMAGE" - // Name of the secret containing default credentials for external container registries. + // RadixExternalRegistryDefaultAuthEnvironmentVariable Name of the secret containing default credentials for external container registries. // Used when pulling images for components and jobs and for pulling images in Dockerfiles when building with buildah. RadixExternalRegistryDefaultAuthEnvironmentVariable = "RADIX_EXTERNAL_REGISTRY_DEFAULT_AUTH_SECRET" ) diff --git a/pkg/apis/utils/git/clone.go b/pkg/apis/utils/git/clone.go index d7fbf8af0..fd66f4ba9 100644 --- a/pkg/apis/utils/git/clone.go +++ b/pkg/apis/utils/git/clone.go @@ -6,29 +6,12 @@ import ( "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/securitycontext" + "github.com/equinor/radix-operator/pkg/apis/utils" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) const ( - // InternalContainerPrefix To indicate that this is not for user interest - InternalContainerPrefix = "internal-" - - // CloneConfigContainerName Name of container for clone in the outer pipeline - CloneConfigContainerName = "clone-config" - - // CloneContainerName Name of container - CloneContainerName = "clone" - - // GitSSHKeyVolumeName Deploy key + known_hosts - GitSSHKeyVolumeName = "git-ssh-keys" - - // BuildContextVolumeName Name of volume to hold build context - BuildContextVolumeName = "build-context" - - // Workspace Folder to hold the code to build - Workspace = "/workspace" - // The script to ensure that github responds before cloning. It breaks after max attempts waitForGithubToRespond = "n=1;max=10;delay=2;while true; do if [ \"$n\" -lt \"$max\" ]; then nslookup github.com && break; n=$((n+1)); sleep $(($delay*$n)); else echo \"The command has failed after $n attempts.\"; break; fi done" ) @@ -41,25 +24,29 @@ type CloneConfig struct { // CloneInitContainers The sidecars for cloning repo func CloneInitContainers(sshURL, branch string, config CloneConfig) []corev1.Container { - return CloneInitContainersWithContainerName(sshURL, branch, CloneContainerName, config) + return CloneInitContainersWithContainerName(sshURL, branch, CloneContainerName, config, true) } // CloneInitContainersWithContainerName The sidecars for cloning repo -func CloneInitContainersWithContainerName(sshURL, branch, cloneContainerName string, config CloneConfig) []corev1.Container { - gitCloneCmd := []string{"git", "clone", "--recurse-submodules", sshURL, "-b", branch, "--verbose", "--progress", Workspace} +func CloneInitContainersWithContainerName(sshURL, branch, cloneContainerName string, config CloneConfig, useLfs bool) []corev1.Container { + gitConfigCommand := fmt.Sprintf("git config --global --add safe.directory %s", Workspace) + gitCloneCommand := fmt.Sprintf("git clone --recurse-submodules %s -b %s --verbose --progress %s", sshURL, branch, Workspace) + getLfsFilesCommands := fmt.Sprintf("cd %s && if [ -n \"$(git lfs ls-files 2>/dev/null)\" ]; then git lfs install && echo 'Pulling large files...' && git lfs pull && echo 'Done'; fi && cd -", Workspace) + gitCloneCmd := []string{"sh", "-c", fmt.Sprintf("%s && %s %s", gitConfigCommand, gitCloneCommand, + utils.TernaryString(useLfs, fmt.Sprintf("&& %s", getLfsFilesCommands), ""))} containers := []corev1.Container{ { - Name: fmt.Sprintf("%snslookup", InternalContainerPrefix), - Image: config.NSlookupImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{waitForGithubToRespond}, - Command: []string{"/bin/sh", "-c"}, + Name: fmt.Sprintf("%snslookup", InternalContainerPrefix), + Image: config.NSlookupImage, + Command: []string{"/bin/sh", "-c"}, + Args: []string{waitForGithubToRespond}, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: *resource.NewScaledQuantity(10, resource.Milli), corev1.ResourceMemory: *resource.NewScaledQuantity(1, resource.Mega), }, }, + ImagePullPolicy: corev1.PullIfNotPresent, SecurityContext: securitycontext.Container( securitycontext.WithContainerRunAsUser(1000), // Any user will probably do securitycontext.WithContainerRunAsGroup(1000), @@ -73,6 +60,12 @@ func CloneInitContainersWithContainerName(sshURL, branch, cloneContainerName str Image: config.GitImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: gitCloneCmd, + Env: []corev1.EnvVar{ + { + Name: "HOME", + Value: CloneRepoHomeVolumePath, + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: BuildContextVolumeName, @@ -83,6 +76,11 @@ func CloneInitContainersWithContainerName(sshURL, branch, cloneContainerName str MountPath: "/.ssh", ReadOnly: true, }, + { + Name: CloneRepoHomeVolumeName, + MountPath: CloneRepoHomeVolumePath, + ReadOnly: false, + }, }, SecurityContext: securitycontext.Container( securitycontext.WithContainerRunAsUser(65534), // Must be this user when running as non root diff --git a/pkg/apis/utils/git/clone_test.go b/pkg/apis/utils/git/clone_test.go index d0ca70afa..d01eb0307 100644 --- a/pkg/apis/utils/git/clone_test.go +++ b/pkg/apis/utils/git/clone_test.go @@ -33,7 +33,7 @@ func Test_CloneInitContainersWithContainerName_CustomImages(t *testing.T) { } cloneName := "anyclonename" cfg := git.CloneConfig{NSlookupImage: "anynslookup:any", GitImage: "anygit:any", BashImage: "anybash:any"} - containers := git.CloneInitContainersWithContainerName("anysshurl", "anybranch", cloneName, cfg) + containers := git.CloneInitContainersWithContainerName("anysshurl", "anybranch", cloneName, cfg, true) actual := slice.Map(containers, func(c v1.Container) containerInfo { return containerInfo{name: c.Name, image: c.Image} }) expected := []containerInfo{ {name: fmt.Sprintf("%snslookup", git.InternalContainerPrefix), image: cfg.NSlookupImage}, diff --git a/pkg/apis/utils/git/defaults.go b/pkg/apis/utils/git/defaults.go new file mode 100644 index 000000000..25ba93c92 --- /dev/null +++ b/pkg/apis/utils/git/defaults.go @@ -0,0 +1,27 @@ +package git + +const ( + // InternalContainerPrefix To indicate that this is not for user interest + InternalContainerPrefix = "internal-" + + // CloneConfigContainerName Name of container for clone in the outer pipeline + CloneConfigContainerName = "clone-config" + + // CloneContainerName Name of container + CloneContainerName = "clone" + + // GitSSHKeyVolumeName Deploy key + known_hosts + GitSSHKeyVolumeName = "git-ssh-keys" + + // BuildContextVolumeName Name of volume to hold build context + BuildContextVolumeName = "build-context" + + // CloneRepoHomeVolumeName Name of volume to hold clone repo home folder + CloneRepoHomeVolumeName = "builder-home" + + // CloneRepoHomeVolumePath Name of home volume path + CloneRepoHomeVolumePath = "/home/clone" + + // Workspace Folder to hold the code to build + Workspace = "/workspace" +)