From 417b9729bbf962e20722cb3ab81b4947a7781d97 Mon Sep 17 00:00:00 2001 From: Jithin Emmanuel Date: Thu, 17 Dec 2020 13:19:42 -0800 Subject: [PATCH] fix(2294): Ability to mount custom volumes. (#56) * fix(2294): Ability to mount custom volumes. allow shared commands to work in non-root containers * fix: tests * fix: tests * Update README.md Co-authored-by: kkawahar Co-authored-by: kkawahar --- README.md | 2 +- cmd/build.go | 8 ++++++++ cmd/build_test.go | 1 + cmd/root_test.go | 1 + go.mod | 3 ++- go.sum | 8 ++++++-- launch/docker.go | 12 ++++++++++-- launch/docker_test.go | 3 ++- launch/launch.go | 14 +++++++++----- launch/launch_test.go | 11 ++++++----- 10 files changed, 46 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 3e03db0..fdf3c54 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ Flags: ex) git@github.com:/.git[#] https://github.com//.git[#] --sudo Use sudo command for container runtime. + --vol string Mount local volumes into build container. (:) (default []) Global Flags: -v, --verbose verbose output. @@ -185,4 +186,3 @@ Code licensed under the BSD 3-Clause license. See [LICENSE](https://github.com/s [build-url]: https://cd.screwdriver.cd/pipelines/4014 [goreport-image]: https://goreportcard.com/badge/github.com/Screwdriver-cd/sd-local [goreport-url]: https://goreportcard.com/report/github.com/Screwdriver-cd/sd-local - diff --git a/cmd/build.go b/cmd/build.go index a14a5f5..2efd337 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -60,6 +60,7 @@ func newBuildCmd() *cobra.Command { var optionMeta string var metaFilePath string var socketPath string + var localVolumes []string buildCmd := &cobra.Command{ Use: "build [job name]", @@ -203,6 +204,7 @@ func newBuildCmd() *cobra.Command { InteractiveMode: interactiveMode, SocketPath: socketPath, FlagVerbose: flagVerbose, + LocalVolumes: localVolumes, } launch := launchNew(option) @@ -298,5 +300,11 @@ ex) git@github.com:/.git[#] launch.DefaultSocketPath(), "Path to the socket. It will used in build container.") + buildCmd.Flags().StringSliceVar( + &localVolumes, + "vol", + []string{}, + "Volumes to mount into build container.") + return buildCmd } diff --git a/cmd/build_test.go b/cmd/build_test.go index 40ade43..94560a3 100644 --- a/cmd/build_test.go +++ b/cmd/build_test.go @@ -30,6 +30,7 @@ Flags: ex) git@github.com:/.git[#] https://github.com//.git[#] --sudo Use sudo command for container runtime. + --vol strings Volumes to mount into build container. ` diff --git a/cmd/root_test.go b/cmd/root_test.go index b0c6951..40e789e 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -120,6 +120,7 @@ Flags: ex) git@github.com:/.git[#] https://github.com//.git[#] --sudo Use sudo command for container runtime. + --vol strings Volumes to mount into build container. Global Flags: -v, --verbose verbose output. diff --git a/go.mod b/go.mod index 6b7596c..285f883 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/blang/semver v3.5.1+incompatible github.com/creack/pty v1.1.11 + github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-yaml/yaml v2.1.0+incompatible github.com/joho/godotenv v1.3.0 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect @@ -17,7 +18,7 @@ require ( github.com/stretchr/testify v1.5.0 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect - golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 // indirect + golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum index cd268a5..0b2346d 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -60,6 +62,7 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -165,8 +168,9 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 h1:c1Sgqkh8v6ZxafNGG64r8C8UisIW2TKMJN8P86tKjr0= -golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM= +golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/launch/docker.go b/launch/docker.go index 71875a3..c291cce 100644 --- a/launch/docker.go +++ b/launch/docker.go @@ -30,6 +30,7 @@ type docker struct { flagVerbose bool interact Interacter socketPath string + localVolumes []string } var _ runner = (*docker)(nil) @@ -45,7 +46,7 @@ const ( orgRepo = "sd-local/local-build" ) -func newDocker(setupImage, setupImageVer string, useSudo bool, interactiveMode bool, socketPath string, flagVerbose bool) runner { +func newDocker(setupImage, setupImageVer string, useSudo bool, interactiveMode bool, socketPath string, flagVerbose bool, localVolumes []string) runner { return &docker{ volume: "SD_LAUNCH_BIN", habVolume: "SD_LAUNCH_HAB", @@ -58,6 +59,7 @@ func newDocker(setupImage, setupImageVer string, useSudo bool, interactiveMode b flagVerbose: flagVerbose, interact: &Interact{}, socketPath: socketPath, + localVolumes: localVolumes, } } @@ -102,6 +104,8 @@ func (d *docker) runBuild(buildEntry buildEntry) error { binVol := fmt.Sprintf("%s:%s", d.volume, "/opt/sd") habVol := fmt.Sprintf("%s:%s", d.habVolume, "/opt/sd/hab") + dockerVolumes := append(d.localVolumes, srcVol, artVol, binVol, habVol, fmt.Sprintf("%s:/tmp/auth.sock", d.socketPath)) + // Overwrite steps for sd-local interact mode. The env will load later. if d.interactiveMode { buildEntry.Steps = []screwdriver.Step{ @@ -124,7 +128,11 @@ func (d *docker) runBuild(buildEntry buildEntry) error { } dockerCommandArgs := []string{"container", "run"} - dockerCommandOptions := []string{"--rm", "-v", srcVol, "-v", artVol, "-v", binVol, "-v", habVol, "-v", fmt.Sprintf("%s:/tmp/auth.sock", d.socketPath), "-e", "SSH_AUTH_SOCK=/tmp/auth.sock", buildImage} + dockerCommandOptions := []string{"--rm"} + for _, v := range dockerVolumes { + dockerCommandOptions = append(dockerCommandOptions, "-v", v) + } + dockerCommandOptions = append(dockerCommandOptions, "-e", "SSH_AUTH_SOCK=/tmp/auth.sock", buildImage) configJSONArg := string(configJSON) if d.interactiveMode { configJSONArg = fmt.Sprintf("%q", configJSONArg) diff --git a/launch/docker_test.go b/launch/docker_test.go index 3ae1a7a..00bd9ae 100644 --- a/launch/docker_test.go +++ b/launch/docker_test.go @@ -63,9 +63,10 @@ func TestNewDocker(t *testing.T) { flagVerbose: false, interact: &Interact{}, socketPath: "/auth.sock", + localVolumes: []string{"path:path"}, } - d := newDocker("launcher", "latest", false, false, "/auth.sock", false) + d := newDocker("launcher", "latest", false, false, "/auth.sock", false, []string{"path:path"}) assert.Equal(t, expected, d) }) diff --git a/launch/launch.go b/launch/launch.go index fcf03d9..6028ca2 100644 --- a/launch/launch.go +++ b/launch/launch.go @@ -64,6 +64,7 @@ type buildEntry struct { InteractiveMode bool `json:"-"` SocketPath string `json:"-"` UsePrivileged bool `json:"-"` + LocalVolumes []string `json:"-"` } // Option is option for launch New @@ -82,6 +83,7 @@ type Option struct { InteractiveMode bool SocketPath string FlagVerbose bool + LocalVolumes []string } const ( @@ -131,10 +133,11 @@ func createBuildEntry(option Option) buildEntry { } defaultEnv := EnvVar{ - "SD_TOKEN": option.JWT, - "SD_ARTIFACTS_DIR": defaultArtDir, - "SD_API_URL": apiURL, - "SD_STORE_URL": storeURL, + "SD_TOKEN": option.JWT, + "SD_ARTIFACTS_DIR": defaultArtDir, + "SD_API_URL": apiURL, + "SD_STORE_URL": storeURL, + "SD_BASE_COMMAND_PATH": "/sd/commands/", } env := mergeEnv(defaultEnv, option.Job.Environment, option.OptionEnv) @@ -157,6 +160,7 @@ func createBuildEntry(option Option) buildEntry { InteractiveMode: option.InteractiveMode, SocketPath: option.SocketPath, UsePrivileged: option.UsePrivileged, + LocalVolumes: option.LocalVolumes, } } @@ -164,7 +168,7 @@ func createBuildEntry(option Option) buildEntry { func New(option Option) Launcher { l := new(launch) - l.runner = newDocker(option.Entry.Launcher.Image, option.Entry.Launcher.Version, option.UseSudo, option.InteractiveMode, option.SocketPath, option.FlagVerbose) + l.runner = newDocker(option.Entry.Launcher.Image, option.Entry.Launcher.Version, option.UseSudo, option.InteractiveMode, option.SocketPath, option.FlagVerbose, option.LocalVolumes) l.buildEntry = createBuildEntry(option) return l diff --git a/launch/launch_test.go b/launch/launch_test.go index 3eb391b..50e6990 100644 --- a/launch/launch_test.go +++ b/launch/launch_test.go @@ -25,11 +25,12 @@ func newBuildEntry(options ...func(b *buildEntry)) buildEntry { b := buildEntry{ ID: 0, Environment: []EnvVar{{ - "SD_ARTIFACTS_DIR": "/test/artifacts", - "SD_API_URL": "http://api-test.screwdriver.cd/v4", - "SD_STORE_URL": "http://store-test.screwdriver.cd/v1", - "SD_TOKEN": "testjwt", - "FOO": "foo", + "SD_ARTIFACTS_DIR": "/test/artifacts", + "SD_API_URL": "http://api-test.screwdriver.cd/v4", + "SD_STORE_URL": "http://store-test.screwdriver.cd/v1", + "SD_TOKEN": "testjwt", + "SD_BASE_COMMAND_PATH": "/sd/commands/", + "FOO": "foo", }}, EventID: 0, JobID: 0,