feat: Add build option no-image-pull (#92)

Co-authored-by: Yuta Kaneda <yukaneda@yahoo-corp.jp>

README.md

@@ -59,6 +59,7 @@ Flags:
   -m, --memory string          Memory limit for build container, which take a positive integer, followed by a suffix of b, k, m, g.
       --meta string            Metadata to pass into the build environment, which is represented with JSON format
       --meta-file string       Path to the meta file. meta file is represented with JSON format.
+      --no-image-pull          Skip container image pulls to save time.
       --privileged             Use privileged mode for container runtime.
   -S, --socket string          Path to the socket. It will used in build container.
       --src-url string         Specify the source url to build.

cmd/build.go

@@ -70,6 +70,7 @@ func newBuildCmd() *cobra.Command {
 	var socketPath string
 	var localVolumes []string
 	var buildUser string
+	var noImagePull bool
 
 	buildCmd := &cobra.Command{
 		Use:   "build [job name]",
@@ -250,6 +251,7 @@ func newBuildCmd() *cobra.Command {
 				FlagVerbose:     flagVerbose,
 				LocalVolumes:    localVolumes,
 				BuildUser:       buildUser,
+				NoImagePull:     noImagePull,
 			}
 
 			launch := launchNew(option)
@@ -358,5 +360,11 @@ ex) git@github.com:<org>/<repo>.git[#<branch>]
 		"",
 		"Change default build user. Default value is from container in use.")
 
+	buildCmd.Flags().BoolVar(
+		&noImagePull,
+		"no-image-pull",
+		false,
+		"Skip container image pulls to save time.")
+
 	return buildCmd
 }

cmd/root_test.go

@@ -54,6 +54,7 @@ Flags:
   -m, --memory string          Memory limit for build container, which take a positive integer, followed by a suffix of b, k, m, g.
       --meta string            Metadata to pass into the build environment, which is represented with JSON format
       --meta-file string       Path to the meta file. meta file is represented with JSON format.
+      --no-image-pull          Skip container image pulls to save time.
       --privileged             Use privileged mode for container runtime.
   -S, --socket string          Path to the socket. It will used in build container.%s
       --src-url string         Specify the source url to build.

launch/docker.go

@@ -43,6 +43,7 @@ type docker struct {
 	socketPath        string
 	localVolumes      []string
 	buildUser         string
+	noImagePull       bool
 	dind              DinD
 }
 
@@ -59,7 +60,7 @@ const (
 	orgRepo = "sd-local/local-build"
 )
 
-func newDocker(setupImage, setupImageVer string, useSudo bool, interactiveMode bool, socketPath string, flagVerbose bool, localVolumes []string, buildUser string, dindEnabled bool) runner {
+func newDocker(setupImage, setupImageVer string, useSudo bool, interactiveMode bool, socketPath string, flagVerbose bool, localVolumes []string, buildUser string, noImagePull bool, dindEnabled bool) runner {
 	return &docker{
 		volume:            "SD_LAUNCH_BIN",
 		habVolume:         "SD_LAUNCH_HAB",
@@ -74,6 +75,7 @@ func newDocker(setupImage, setupImageVer string, useSudo bool, interactiveMode b
 		socketPath:        socketPath,
 		localVolumes:      localVolumes,
 		buildUser:         buildUser,
+		noImagePull:       noImagePull,
 		dind: DinD{
 			enabled:         dindEnabled,
 			volume:          "SD_DIND_CERT",
@@ -90,17 +92,20 @@ func (d *docker) setupBin() error {
 	mount := fmt.Sprintf("%s:/opt/sd/", d.volume)
 	habMount := fmt.Sprintf("%s:/hab", d.habVolume)
 	image := fmt.Sprintf("%s:%s", d.setupImage, d.setupImageVersion)
-	_, err := d.execDockerCommand("pull", image)
-	if err != nil {
-		return fmt.Errorf("failed to pull launcher image: %v", err)
+
+	if !d.noImagePull {
+		_, err := d.execDockerCommand("pull", image)
+		if err != nil {
+			return fmt.Errorf("failed to pull launcher image: %v", err)
+		}
 	}
 
 	// The mechanism for population is that VOLUMEs were declared in the image, so they copy what was in their layer to
 	// the mounted location on first mount of non-existing volumes
 	// NOTE: docker allows copying to first-time mounted as well, but both docker and podman copy to non-existing ones.
 	//       therefore, volumes are not pre-created, but created on first mention by the image that populates them
 	//       and then used by subsequent images that then use their content.
-	_, err = d.execDockerCommand("container", "run", "--rm", "-v", mount, "-v", habMount, "--entrypoint", "/bin/echo", image, "set up bin")
+	_, err := d.execDockerCommand("container", "run", "--rm", "--pull", "never", "-v", mount, "-v", habMount, "--entrypoint", "/bin/echo", image, "set up bin")
 	if err != nil {
 		return fmt.Errorf("failed to prepare build scripts: %v", err)
 	}
@@ -145,14 +150,17 @@ func (d *docker) runBuild(buildEntry buildEntry) error {
 		return err
 	}
 
-	logrus.Infof("Pulling docker image from %s...", buildImage)
-	_, err = d.execDockerCommand("pull", buildImage)
-	if err != nil {
-		return fmt.Errorf("failed to pull user image %v", err)
+	if !d.noImagePull {
+		logrus.Infof("Pulling docker image from %s...", buildImage)
+		_, err = d.execDockerCommand("pull", buildImage)
+		if err != nil {
+			return fmt.Errorf("failed to pull user image %v", err)
+		}
 	}
 
 	dockerCommandArgs := []string{"container", "run"}
 	dockerCommandOptions := []string{"--rm"}
+	dockerCommandOptions = append(dockerCommandOptions, "--pull", "never")
 	for _, v := range dockerVolumes {
 		dockerCommandOptions = append(dockerCommandOptions, "-v", v)
 	}
@@ -229,10 +237,12 @@ func (d *docker) runBuild(buildEntry buildEntry) error {
 }
 
 func (d *docker) runDinD() error {
-	logrus.Infof("Pulling dind image from %s...", d.dind.image)
-	_, err := d.execDockerCommand("pull", d.dind.image)
-	if err != nil {
-		return fmt.Errorf("failed to pull user image %v", err)
+	if !d.noImagePull {
+		logrus.Infof("Pulling dind image from %s...", d.dind.image)
+		_, err := d.execDockerCommand("pull", d.dind.image)
+		if err != nil {
+			return fmt.Errorf("failed to pull user image %v", err)
+		}
 	}
 
 	if _, err := d.execDockerCommand([]string{"network", "create", d.dind.network}...); err != nil {
@@ -243,6 +253,7 @@ func (d *docker) runDinD() error {
 	dockerCommandOptions := []string{
 		"--rm",
 		"--privileged",
+		"--pull", "never",
 		"--name", "sd-local-dind",
 		"-d",
 		"--network", d.dind.network,

launch/docker_test.go

@@ -67,6 +67,7 @@ func TestNewDocker(t *testing.T) {
 			socketPath:        "/auth.sock",
 			localVolumes:      []string{"path:path"},
 			buildUser:         "jithin",
+			noImagePull:       false,
 			dind: DinD{
 				enabled:         true,
 				volume:          "SD_DIND_CERT",
@@ -78,7 +79,7 @@ func TestNewDocker(t *testing.T) {
 			},
 		}
 
-		d := newDocker("launcher", "latest", false, false, "/auth.sock", false, []string{"path:path"}, "jithin", true)
+		d := newDocker("launcher", "latest", false, false, "/auth.sock", false, []string{"path:path"}, "jithin", false, true)
 
 		assert.Equal(t, expected, d)
 	})
@@ -180,12 +181,12 @@ func TestRunBuild(t *testing.T) {
 		{"success", "SUCCESS_RUN_BUILD", nil,
 			[]string{
 				"docker pull node:12",
-				fmt.Sprintf("docker container run --rm -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.volume, d.habVolume, sshSocket)},
+				fmt.Sprintf("docker container run --rm --pull never -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.volume, d.habVolume, sshSocket)},
 			newBuildEntry()},
 		{"success with memory limit", "SUCCESS_RUN_BUILD", nil,
 			[]string{
 				"docker pull node:12",
-				fmt.Sprintf("docker container run -m2GB --rm -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.volume, d.habVolume, sshSocket)},
+				fmt.Sprintf("docker container run -m2GB --rm --pull never -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.volume, d.habVolume, sshSocket)},
 			newBuildEntry(func(b *buildEntry) {
 				b.MemoryLimit = "2GB"
 			})},
@@ -242,9 +243,65 @@ func TestRunBuildWithDind(t *testing.T) {
 			[]string{
 				"docker pull docker:23.0.1-dind-rootless",
 				"docker network create sd-local-dind-bridge",
-				"docker container run --rm --privileged --name sd-local-dind -d --network sd-local-dind-bridge --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v SD_DIND_CERT:/certs/client -v SD_DIND_SHARE:/opt/sd_dind_share docker:23.0.1-dind-rootless",
+				"docker container run --rm --privileged --pull never --name sd-local-dind -d --network sd-local-dind-bridge --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v SD_DIND_CERT:/certs/client -v SD_DIND_SHARE:/opt/sd_dind_share docker:23.0.1-dind-rootless",
 				"docker pull node:12",
-				fmt.Sprintf("docker container run --network %s -e DOCKER_TLS_CERTDIR=/certs -e DOCKER_HOST=tcp://docker:2376 -e DOCKER_TLS_VERIFY=1 -e DOCKER_CERT_PATH=/certs/client -e SD_DIND_SHARE_PATH=%s -v %s:/certs/client:ro -v %s:%s --rm -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.dind.network, d.dind.shareVolumePath, d.dind.volume, d.dind.shareVolumeName, d.dind.shareVolumePath, d.volume, d.habVolume, sshSocket)},
+				fmt.Sprintf("docker container run --network %s -e DOCKER_TLS_CERTDIR=/certs -e DOCKER_HOST=tcp://docker:2376 -e DOCKER_TLS_VERIFY=1 -e DOCKER_CERT_PATH=/certs/client -e SD_DIND_SHARE_PATH=%s -v %s:/certs/client:ro -v %s:%s --rm --pull never -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.dind.network, d.dind.shareVolumePath, d.dind.volume, d.dind.shareVolumeName, d.dind.shareVolumePath, d.volume, d.habVolume, sshSocket)},
+			newBuildEntry(func(b *buildEntry) {
+				b.Annotations["screwdriver.cd/dockerEnabled"] = true
+			})},
+	}
+
+	for _, tt := range testCase {
+		t.Run(tt.name, func(t *testing.T) {
+			c := newFakeExecCommand(tt.id)
+			execCommand = c.execCmd
+			err := d.runBuild(tt.buildEntry)
+			for i, expectedCommand := range tt.expectedCommands {
+				assert.True(t, strings.Contains(c.commands[i], expectedCommand), "expect %q \nbut got \n%q", expectedCommand, c.commands[i])
+			}
+			if tt.expectError != nil {
+				assert.Equal(t, tt.expectError.Error(), err.Error())
+			} else {
+				assert.Nil(t, err)
+			}
+		})
+	}
+}
+
+func TestRunBuildWithNoImagePull(t *testing.T) {
+	defer func() {
+		execCommand = exec.Command
+	}()
+
+	d := &docker{
+		volume:            "SD_LAUNCH_BIN",
+		setupImage:        "launcher",
+		setupImageVersion: "latest",
+		socketPath:        os.Getenv("SSH_AUTH_SOCK"),
+		noImagePull:       true,
+		dind: DinD{
+			enabled:         true,
+			volume:          "SD_DIND_CERT",
+			shareVolumeName: "SD_DIND_SHARE",
+			shareVolumePath: "/opt/sd_dind_share",
+			container:       "sd-local-dind",
+			network:         "sd-local-dind-bridge",
+			image:           "docker:23.0.1-dind-rootless",
+		},
+	}
+
+	testCase := []struct {
+		name             string
+		id               string
+		expectError      error
+		expectedCommands []string
+		buildEntry       buildEntry
+	}{
+		{"success with no image pull", "SUCCESS_RUN_BUILD", nil,
+			[]string{
+				"docker network create sd-local-dind-bridge",
+				"docker container run --rm --privileged --pull never --name sd-local-dind -d --network sd-local-dind-bridge --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v SD_DIND_CERT:/certs/client -v SD_DIND_SHARE:/opt/sd_dind_share docker:23.0.1-dind-rootless",
+				fmt.Sprintf("docker container run --network %s -e DOCKER_TLS_CERTDIR=/certs -e DOCKER_HOST=tcp://docker:2376 -e DOCKER_TLS_VERIFY=1 -e DOCKER_CERT_PATH=/certs/client -e SD_DIND_SHARE_PATH=%s -v %s:/certs/client:ro -v %s:%s --rm --pull never -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.dind.network, d.dind.shareVolumePath, d.dind.volume, d.dind.shareVolumeName, d.dind.shareVolumePath, d.volume, d.habVolume, sshSocket)},
 			newBuildEntry(func(b *buildEntry) {
 				b.Annotations["screwdriver.cd/dockerEnabled"] = true
 			})},
@@ -299,12 +356,12 @@ func TestRunBuildWithSudo(t *testing.T) {
 		{"success", "SUCCESS_RUN_BUILD_SUDO", nil,
 			[]string{
 				"sudo docker pull node:12",
-				fmt.Sprintf("sudo docker container run --rm -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.volume, d.habVolume, sshSocket)},
+				fmt.Sprintf("sudo docker container run --rm --pull never -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.volume, d.habVolume, sshSocket)},
 			newBuildEntry()},
 		{"success with memory limit", "SUCCESS_RUN_BUILD_SUDO", nil,
 			[]string{
 				"sudo docker pull node:12",
-				fmt.Sprintf("sudo docker container run -m2GB --rm -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.volume, d.habVolume, sshSocket)},
+				fmt.Sprintf("sudo docker container run -m2GB --rm --pull never -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.volume, d.habVolume, sshSocket)},
 			newBuildEntry(func(b *buildEntry) {
 				b.MemoryLimit = "2GB"
 			})},
@@ -363,13 +420,13 @@ func TestRunBuildWithInteractiveMode(t *testing.T) {
 		{"success", "SUCCESS_RUN_BUILD_INTERACT", nil,
 			[]string{
 				"sudo docker pull node:12",
-				fmt.Sprintf("sudo docker container run -itd --rm -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12", d.volume, d.habVolume, sshSocket),
+				fmt.Sprintf("sudo docker container run -itd --rm --pull never -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12", d.volume, d.habVolume, sshSocket),
 				"sudo docker attach "},
 			newBuildEntry()},
 		{"success with memory limit", "SUCCESS_RUN_BUILD_INTERACT", nil,
 			[]string{
 				"sudo docker pull node:12",
-				fmt.Sprintf("sudo docker container run -m2GB -itd --rm -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12", d.volume, d.habVolume, sshSocket),
+				fmt.Sprintf("sudo docker container run -m2GB -itd --rm --pull never -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12", d.volume, d.habVolume, sshSocket),
 				"sudo docker attach SUCCESS_RUN_BUILD_INTERACT"},
 			newBuildEntry(func(b *buildEntry) {
 				b.MemoryLimit = "2GB"

launch/launch.go

@@ -83,6 +83,7 @@ type Option struct {
 	FlagVerbose     bool
 	LocalVolumes    []string
 	BuildUser       string
+	NoImagePull     bool
 }
 
 const (
@@ -153,7 +154,7 @@ func New(option Option) Launcher {
 	l := new(launch)
 	dindEnabled, _ := option.Job.Annotations["screwdriver.cd/dockerEnabled"].(bool)
 
-	l.runner = newDocker(option.Entry.Launcher.Image, option.Entry.Launcher.Version, option.UseSudo, option.InteractiveMode, option.SocketPath, option.FlagVerbose, option.LocalVolumes, option.BuildUser, dindEnabled)
+	l.runner = newDocker(option.Entry.Launcher.Image, option.Entry.Launcher.Version, option.UseSudo, option.InteractiveMode, option.SocketPath, option.FlagVerbose, option.LocalVolumes, option.BuildUser, option.NoImagePull, dindEnabled)
 	l.buildEntry = createBuildEntry(option)
 
 	return l