diff --git a/README.md b/README.md index b8e3e64..5832d35 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,15 @@ Global Flags: 1. environment in a screwdriver.yaml 1. defaultEnv (e.g.: `SD_TOKEN`, `SD_API_URL`) +* You can use docker commands in the build to run containers, build images, etc. + * Set `screwdriver.cd/dockerEnabled: true` in the job annotations. +```yaml +jobs: + main: + annotations: + screwdriver.cd/dockerEnabled: true +``` + ##### config _create_ ```bash diff --git a/launch/docker.go b/launch/docker.go index 4fc4c94..35b5cc4 100644 --- a/launch/docker.go +++ b/launch/docker.go @@ -18,6 +18,16 @@ import ( "github.com/sirupsen/logrus" ) +// DinD has the information needed to start the dind-rootless container +type DinD struct { + volume string + shareVolumeName string + shareVolumePath string + container string + network string + image string +} + type docker struct { volume string habVolume string @@ -32,6 +42,7 @@ type docker struct { socketPath string localVolumes []string buildUser string + dind DinD } var _ runner = (*docker)(nil) @@ -62,6 +73,14 @@ func newDocker(setupImage, setupImageVer string, useSudo bool, interactiveMode b socketPath: socketPath, localVolumes: localVolumes, buildUser: buildUser, + dind: DinD{ + volume: "SD_DIND_CERT", + shareVolumeName: "SD_DIND_SHARE", + shareVolumePath: "/opt/sd_dind_share", + container: "sd-local-dind", + network: "sd-local-dind-bridge", + image: "docker:23.0.1-dind-rootless", + }, } } @@ -88,6 +107,14 @@ func (d *docker) setupBin() error { } func (d *docker) runBuild(buildEntry buildEntry) error { + dockerEnabled, _ := buildEntry.Annotations["screwdriver.cd/dockerEnabled"].(bool) + + if dockerEnabled { + if err := d.runDinD(); err != nil { + return fmt.Errorf("failed to prepare dind container: %v", err) + } + } + environment := buildEntry.Environment srcDir := buildEntry.SrcPath @@ -151,6 +178,21 @@ func (d *docker) runBuild(buildEntry buildEntry) error { dockerCommandOptions = append([]string{"--privileged"}, dockerCommandOptions...) } + if dockerEnabled { + dockerCommandOptions = append( + []string{ + "--network", d.dind.network, + "-e", "DOCKER_TLS_CERTDIR=/certs", + "-e", "DOCKER_HOST=tcp://docker:2376", + "-e", "DOCKER_TLS_VERIFY=1", + "-e", "DOCKER_CERT_PATH=/certs/client", + "-e", fmt.Sprintf("SD_DIND_SHARE_PATH=%s", d.dind.shareVolumePath), + "-v", fmt.Sprintf("%s:/certs/client:ro", d.dind.volume), + "-v", fmt.Sprintf("%s:%s", d.dind.shareVolumeName, d.dind.shareVolumePath), + }, + dockerCommandOptions...) + } + if d.buildUser != "" { dockerCommandOptions = append([]string{fmt.Sprintf("-u%s", d.buildUser)}, dockerCommandOptions...) } @@ -186,6 +228,38 @@ func (d *docker) runBuild(buildEntry buildEntry) error { return nil } +func (d *docker) runDinD() error { + logrus.Infof("Pulling dind image from %s...", d.dind.image) + _, err := d.execDockerCommand("pull", d.dind.image) + if err != nil { + return fmt.Errorf("failed to pull user image %v", err) + } + + if _, err := d.execDockerCommand([]string{"network", "create", d.dind.network}...); err != nil { + return fmt.Errorf("failed to create network: %v", err) + } + + dockerCommandArgs := []string{"container", "run"} + dockerCommandOptions := []string{ + "--rm", + "--privileged", + "--name", "sd-local-dind", + "-d", + "--network", d.dind.network, + "--network-alias", "docker", + "-e", "DOCKER_TLS_CERTDIR=/certs", + "-v", fmt.Sprintf("%s:/certs/client", d.dind.volume), + "-v", fmt.Sprintf("%s:/opt/sd_dind_share", d.dind.shareVolumeName), + d.dind.image, + } + + if _, err := d.execDockerCommand(append(dockerCommandArgs, dockerCommandOptions...)...); err != nil { + return fmt.Errorf("failed to run dind container: %v", err) + } + + return nil +} + func (d *docker) attachDockerCommand(attachCommands []string, commands [][]string) error { attachCommands = append([]string{"docker"}, attachCommands...) if d.useSudo { @@ -268,6 +342,30 @@ func (d *docker) clean() { if err != nil { logrus.Warn(fmt.Errorf("failed to remove hab volume: %v", err)) } + + _, err = d.execDockerCommand("kill", d.dind.container) + + if err != nil { + logrus.Warn(fmt.Errorf("failed to remove dind container: %v", err)) + } + + _, err = d.execDockerCommand("network", "rm", "--force", d.dind.network) + + if err != nil { + logrus.Warn(fmt.Errorf("failed to remove dind volume: %v", err)) + } + + _, err = d.execDockerCommand("volume", "rm", "--force", d.dind.volume) + + if err != nil { + logrus.Warn(fmt.Errorf("failed to remove dind volume: %v", err)) + } + + _, err = d.execDockerCommand("volume", "rm", "--force", d.dind.shareVolumeName) + + if err != nil { + logrus.Warn(fmt.Errorf("failed to remove dind share volume: %v", err)) + } } func (d *docker) waitForProcess(cmds []*exec.Cmd) error { diff --git a/launch/docker_test.go b/launch/docker_test.go index 672d430..6d18c4f 100644 --- a/launch/docker_test.go +++ b/launch/docker_test.go @@ -67,6 +67,14 @@ func TestNewDocker(t *testing.T) { socketPath: "/auth.sock", localVolumes: []string{"path:path"}, buildUser: "jithin", + dind: DinD{ + volume: "SD_DIND_CERT", + shareVolumeName: "SD_DIND_SHARE", + shareVolumePath: "/opt/sd_dind_share", + container: "sd-local-dind", + network: "sd-local-dind-bridge", + image: "docker:23.0.1-dind-rootless", + }, } d := newDocker("launcher", "latest", false, false, "/auth.sock", false, []string{"path:path"}, "jithin") @@ -150,6 +158,14 @@ func TestRunBuild(t *testing.T) { setupImage: "launcher", setupImageVersion: "latest", socketPath: os.Getenv("SSH_AUTH_SOCK"), + dind: DinD{ + volume: "SD_DIND_CERT", + shareVolumeName: "SD_DIND_SHARE", + shareVolumePath: "/opt/sd_dind_share", + container: "sd-local-dind", + network: "sd-local-dind-bridge", + image: "docker:23.0.1-dind-rootless", + }, } testCase := []struct { @@ -171,6 +187,16 @@ func TestRunBuild(t *testing.T) { newBuildEntry(func(b *buildEntry) { b.MemoryLimit = "2GB" })}, + {"success with dind", "SUCCESS_RUN_BUILD", nil, + []string{ + "docker pull docker:23.0.1-dind-rootless", + "docker network create sd-local-dind-bridge", + "docker container run --rm --privileged --name sd-local-dind -d --network sd-local-dind-bridge --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v SD_DIND_CERT:/certs/client -v SD_DIND_SHARE:/opt/sd_dind_share docker:23.0.1-dind-rootless", + "docker pull node:12", + fmt.Sprintf("docker container run --network %s -e DOCKER_TLS_CERTDIR=/certs -e DOCKER_HOST=tcp://docker:2376 -e DOCKER_TLS_VERIFY=1 -e DOCKER_CERT_PATH=/certs/client -e SD_DIND_SHARE_PATH=%s -v %s:/certs/client:ro -v %s:%s --rm -v /:/sd/workspace/src/screwdriver.cd/sd-local/local-build -v sd-artifacts/:/test/artifacts -v %s:/opt/sd -v %s:/opt/sd/hab -v %s --entrypoint /bin/sh -e SSH_AUTH_SOCK=/tmp/auth.sock node:12 /opt/sd/local_run.sh ", d.dind.network, d.dind.shareVolumePath, d.dind.volume, d.dind.shareVolumeName, d.dind.shareVolumePath, d.volume, d.habVolume, sshSocket)}, + newBuildEntry(func(b *buildEntry) { + b.Annotations["screwdriver.cd/dockerEnabled"] = true + })}, {"failure build run", "FAIL_BUILD_CONTAINER_RUN", fmt.Errorf("failed to run build container: exit status 1"), []string{}, newBuildEntry()}, {"failure build image pull", "FAIL_BUILD_IMAGE_PULL", fmt.Errorf("failed to pull user image exit status 1"), []string{}, newBuildEntry()}, } diff --git a/launch/launch.go b/launch/launch.go index 00219dd..4ca8cb7 100644 --- a/launch/launch.go +++ b/launch/launch.go @@ -44,24 +44,25 @@ type launch struct { type Meta map[string]interface{} type buildEntry struct { - ID int `json:"id"` - Environment []map[string]string `json:"environment"` - EventID int `json:"eventId"` - JobID int `json:"jobId"` - ParentBuildID []int `json:"parentBuildId"` - Sha string `json:"sha"` - Meta Meta `json:"meta"` - Steps []screwdriver.Step `json:"steps"` - Image string `json:"-"` - JobName string `json:"-"` - ArtifactsPath string `json:"-"` - MemoryLimit string `json:"-"` - SrcPath string `json:"-"` - UseSudo bool `json:"-"` - InteractiveMode bool `json:"-"` - SocketPath string `json:"-"` - UsePrivileged bool `json:"-"` - LocalVolumes []string `json:"-"` + ID int `json:"id"` + Environment []map[string]string `json:"environment"` + EventID int `json:"eventId"` + JobID int `json:"jobId"` + ParentBuildID []int `json:"parentBuildId"` + Sha string `json:"sha"` + Meta Meta `json:"meta"` + Annotations map[string]interface{} `json:"annotations"` + Steps []screwdriver.Step `json:"steps"` + Image string `json:"-"` + JobName string `json:"-"` + ArtifactsPath string `json:"-"` + MemoryLimit string `json:"-"` + SrcPath string `json:"-"` + UseSudo bool `json:"-"` + InteractiveMode bool `json:"-"` + SocketPath string `json:"-"` + UsePrivileged bool `json:"-"` + LocalVolumes []string `json:"-"` } // Option is option for launch New @@ -132,6 +133,7 @@ func createBuildEntry(option Option) buildEntry { ParentBuildID: []int{0}, Sha: "dummy", Meta: option.Meta, + Annotations: option.Job.Annotations, Steps: option.Job.Steps, Image: option.Job.Image, JobName: option.JobName, diff --git a/launch/launch_test.go b/launch/launch_test.go index 96e05fc..58f5e67 100644 --- a/launch/launch_test.go +++ b/launch/launch_test.go @@ -30,6 +30,7 @@ func newBuildEntry(options ...func(b *buildEntry)) buildEntry { ParentBuildID: []int{0}, Sha: "dummy", Meta: Meta{}, + Annotations: map[string]interface{}{}, Steps: job.Steps, Image: job.Image, JobName: "test", @@ -49,6 +50,7 @@ func TestNew(t *testing.T) { job := screwdriver.Job{} _ = json.Unmarshal(buf, &job) job.Environment = append(job.Environment, map[string]string{"SD_ARTIFACTS_DIR": "/test/artifacts"}) + job.Annotations = map[string]interface{}{} config := config.Entry{ APIURL: "http://api-test.screwdriver.cd", @@ -81,6 +83,7 @@ func TestNew(t *testing.T) { t.Run("success with default artifacts dir", func(t *testing.T) { buf, _ := ioutil.ReadFile(filepath.Join(testDir, "job.json")) job := screwdriver.Job{} + job.Annotations = map[string]interface{}{} _ = json.Unmarshal(buf, &job) config := config.Entry{ diff --git a/screwdriver/screwdriver.go b/screwdriver/screwdriver.go index 733d633..aa19105 100644 --- a/screwdriver/screwdriver.go +++ b/screwdriver/screwdriver.go @@ -76,9 +76,10 @@ func (en *EnvVars) AppendAll(en2 map[string]string) { // Job is job entity struct type Job struct { - Steps []Step `json:"commands"` - Environment EnvVars `json:"environment"` - Image string `json:"image"` + Annotations map[string]interface{} `json:"annotations"` + Steps []Step `json:"commands"` + Environment EnvVars `json:"environment"` + Image string `json:"image"` } type jobs map[string][]Job