Skip to content

Commit

Permalink
Support local features provides inside .devcontainer dir
Browse files Browse the repository at this point in the history
This allows features to be provided inside the .devcontainer directory
rather than downloaded from a registry, matching the behavior of the
official devcontainer tooling.

For example, Kubernetes tooling generates a devcontainer.json
containing:

    "features": {
        "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
            "enableNonRootDocker": "true",
            "moby": "true"
        },
        "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {},
        "./local-features/copy-kube-config": {}
    },
  • Loading branch information
aaronlehmann committed Mar 2, 2024
1 parent eb069e4 commit b5e15ad
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 32 deletions.
21 changes: 14 additions & 7 deletions devcontainer/devcontainer.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,14 +169,14 @@ func (s *Spec) Compile(fs billy.Filesystem, devcontainerDir, scratchDir, fallbac
if remoteUser == "" {
remoteUser = params.User
}
params.DockerfileContent, err = s.compileFeatures(fs, scratchDir, params.User, remoteUser, params.DockerfileContent)
params.DockerfileContent, err = s.compileFeatures(fs, devcontainerDir, scratchDir, params.User, remoteUser, params.DockerfileContent)
if err != nil {
return nil, err
}
return params, nil
}

func (s *Spec) compileFeatures(fs billy.Filesystem, scratchDir, containerUser, remoteUser, dockerfileContent string) (string, error) {
func (s *Spec) compileFeatures(fs billy.Filesystem, devcontainerDir, scratchDir, containerUser, remoteUser, dockerfileContent string) (string, error) {
// If there are no features, we don't need to do anything!
if len(s.Features) == 0 {
return dockerfileContent, nil
Expand All @@ -200,9 +200,16 @@ func (s *Spec) compileFeatures(fs billy.Filesystem, scratchDir, containerUser, r
sort.Strings(featureOrder)

for _, featureRefRaw := range featureOrder {
featureRefParsed, err := name.NewTag(featureRefRaw)
if err != nil {
return "", fmt.Errorf("parse feature ref %s: %w", featureRefRaw, err)
var (
featureRef string
ok bool
)
if _, featureRef, ok = strings.Cut(featureRefRaw, "./"); !ok {
featureRefParsed, err := name.NewTag(featureRefRaw)
if err != nil {
return "", fmt.Errorf("parse feature ref %s: %w", featureRefRaw, err)
}
featureRef = featureRefParsed.Repository.Name()
}

featureOpts := map[string]any{}
Expand All @@ -222,13 +229,13 @@ func (s *Spec) compileFeatures(fs billy.Filesystem, scratchDir, containerUser, r
// devcontainers/cli has a very complex method of computing the feature
// name from the feature reference. We're just going to hash it for simplicity.
featureSha := md5.Sum([]byte(featureRefRaw))
featureName := filepath.Base(featureRefParsed.Repository.Name())
featureName := filepath.Base(featureRef)
featureDir := filepath.Join(featuresDir, fmt.Sprintf("%s-%x", featureName, featureSha[:4]))
err = fs.MkdirAll(featureDir, 0644)
if err != nil {
return "", err
}
spec, err := features.Extract(fs, featureDir, featureRefRaw)
spec, err := features.Extract(fs, devcontainerDir, featureDir, featureRefRaw)
if err != nil {
return "", fmt.Errorf("extract feature %s: %w", featureRefRaw, err)
}
Expand Down
56 changes: 40 additions & 16 deletions devcontainer/features/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,22 @@ import (
"github.com/go-git/go-billy/v5"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/otiai10/copy"
"github.com/tailscale/hujson"
)

// Extract unpacks the feature from the image and returns the
// parsed specification.
func Extract(fs billy.Filesystem, directory, reference string) (*Spec, error) {
func extractFromImage(fs billy.Filesystem, directory, reference string) error {
ref, err := name.ParseReference(reference)
if err != nil {
return nil, fmt.Errorf("parse feature ref %s: %w", reference, err)
return fmt.Errorf("parse feature ref %s: %w", reference, err)
}
image, err := remote.Image(ref)
if err != nil {
return nil, fmt.Errorf("fetch feature image %s: %w", reference, err)
return fmt.Errorf("fetch feature image %s: %w", reference, err)
}
manifest, err := image.Manifest()
if err != nil {
return nil, fmt.Errorf("fetch feature manifest %s: %w", reference, err)
return fmt.Errorf("fetch feature manifest %s: %w", reference, err)
}

var tarLayer *tar.Reader
Expand All @@ -42,17 +41,17 @@ func Extract(fs billy.Filesystem, directory, reference string) (*Spec, error) {
}
layer, err := image.LayerByDigest(manifestLayer.Digest)
if err != nil {
return nil, fmt.Errorf("fetch feature layer %s: %w", reference, err)
return fmt.Errorf("fetch feature layer %s: %w", reference, err)
}
layerReader, err := layer.Uncompressed()
if err != nil {
return nil, fmt.Errorf("uncompress feature layer %s: %w", reference, err)
return fmt.Errorf("uncompress feature layer %s: %w", reference, err)
}
tarLayer = tar.NewReader(layerReader)
break
}
if tarLayer == nil {
return nil, fmt.Errorf("no tar layer found with media type %q: are you sure this is a devcontainer feature?", TarLayerMediaType)
return fmt.Errorf("no tar layer found with media type %q: are you sure this is a devcontainer feature?", TarLayerMediaType)
}

for {
Expand All @@ -61,35 +60,60 @@ func Extract(fs billy.Filesystem, directory, reference string) (*Spec, error) {
break
}
if err != nil {
return nil, fmt.Errorf("read feature layer %s: %w", reference, err)
return fmt.Errorf("read feature layer %s: %w", reference, err)
}
path := filepath.Join(directory, header.Name)
switch header.Typeflag {
case tar.TypeDir:
err = fs.MkdirAll(path, 0755)
if err != nil {
return nil, fmt.Errorf("mkdir %s: %w", path, err)
return fmt.Errorf("mkdir %s: %w", path, err)
}
case tar.TypeReg:
outFile, err := fs.Create(path)
if err != nil {
return nil, fmt.Errorf("create %s: %w", path, err)
return fmt.Errorf("create %s: %w", path, err)
}
_, err = io.Copy(outFile, tarLayer)
if err != nil {
return nil, fmt.Errorf("copy %s: %w", path, err)
return fmt.Errorf("copy %s: %w", path, err)
}
err = outFile.Close()
if err != nil {
return nil, fmt.Errorf("close %s: %w", path, err)
return fmt.Errorf("close %s: %w", path, err)
}
default:
return nil, fmt.Errorf("unknown type %d in %s", header.Typeflag, path)
return fmt.Errorf("unknown type %d in %s", header.Typeflag, path)
}
}
return nil
}

// Extract unpacks the feature from the image and returns the
// parsed specification.
func Extract(fs billy.Filesystem, devcontainerDir, directory, reference string) (*Spec, error) {
if strings.HasPrefix(reference, "./") {
if err := copy.Copy(filepath.Join(devcontainerDir, reference), directory, copy.Options{
PreserveTimes: true,
PreserveOwner: true,
OnSymlink: func(src string) copy.SymlinkAction {
return copy.Shallow
},
OnError: func(src, dest string, err error) error {
if err == nil {
return nil
}
return fmt.Errorf("copy error: %q -> %q: %w", reference, directory, err)
},
}); err != nil {
return nil, err
}
} else if err := extractFromImage(fs, directory, reference); err != nil {
return nil, err
}

installScriptPath := filepath.Join(directory, "install.sh")
_, err = fs.Stat(installScriptPath)
_, err := fs.Stat(installScriptPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, errors.New("install.sh must be in the root of the feature")
Expand Down
10 changes: 5 additions & 5 deletions devcontainer/features/features_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ func TestExtract(t *testing.T) {
registry := registrytest.New(t)
ref := registrytest.WriteContainer(t, registry, "coder/test:latest", "some/type", nil)
fs := memfs.New()
_, err := features.Extract(fs, "/", ref)
_, err := features.Extract(fs, "", "/", ref)
require.ErrorContains(t, err, "no tar layer found")
})
t.Run("MissingInstallScript", func(t *testing.T) {
Expand All @@ -27,7 +27,7 @@ func TestExtract(t *testing.T) {
"devcontainer-feature.json": "{}",
})
fs := memfs.New()
_, err := features.Extract(fs, "/", ref)
_, err := features.Extract(fs, "", "/", ref)
require.ErrorContains(t, err, "install.sh")
})
t.Run("MissingFeatureFile", func(t *testing.T) {
Expand All @@ -37,7 +37,7 @@ func TestExtract(t *testing.T) {
"install.sh": "hey",
})
fs := memfs.New()
_, err := features.Extract(fs, "/", ref)
_, err := features.Extract(fs, "", "/", ref)
require.ErrorContains(t, err, "devcontainer-feature.json")
})
t.Run("MissingFeatureProperties", func(t *testing.T) {
Expand All @@ -48,7 +48,7 @@ func TestExtract(t *testing.T) {
"devcontainer-feature.json": features.Spec{},
})
fs := memfs.New()
_, err := features.Extract(fs, "/", ref)
_, err := features.Extract(fs, "", "/", ref)
require.ErrorContains(t, err, "id is required")
})
t.Run("Success", func(t *testing.T) {
Expand All @@ -63,7 +63,7 @@ func TestExtract(t *testing.T) {
},
})
fs := memfs.New()
_, err := features.Extract(fs, "/", ref)
_, err := features.Extract(fs, "", "/", ref)
require.NoError(t, err)
})
}
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ require (
github.com/google/go-containerregistry v0.15.2
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/mattn/go-isatty v0.0.19
github.com/otiai10/copy v1.14.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.8.4
Expand Down Expand Up @@ -186,7 +187,6 @@ require (
github.com/opencontainers/runc v1.1.5 // indirect
github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/otiai10/copy v1.12.0 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -725,8 +725,8 @@ github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuh
github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU=
github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4=
github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY=
github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
Expand Down
22 changes: 21 additions & 1 deletion integration/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,18 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) {
"install.sh": "echo $PINEAPPLE > /test2output",
})

feature3Spec, err := json.Marshal(features.Spec{
ID: "test3",
Name: "test3",
Version: "1.0.0",
Options: map[string]features.Option{
"grape": {
Type: "string",
},
},
})
require.NoError(t, err)

// Ensures that a Git repository with a devcontainer.json is cloned and built.
url := createGitServer(t, gitServerOptions{
files: map[string]string{
Expand All @@ -124,10 +136,15 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) {
},
"` + feature2Ref + `": {
"pineapple": "hello from test 2!"
},
"./feature3": {
"grape": "hello from test 3!"
}
}
}`,
".devcontainer/Dockerfile": "FROM ubuntu",
".devcontainer/Dockerfile": "FROM ubuntu",
".devcontainer/feature3/devcontainer-feature.json": string(feature3Spec),
".devcontainer/feature3/install.sh": "echo $GRAPE > /test3output",
},
})
ctr, err := runEnvbuilder(t, options{env: []string{
Expand All @@ -140,6 +157,9 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) {

test2Output := execContainer(t, ctr, "cat /test2output")
require.Equal(t, "hello from test 2!", strings.TrimSpace(test2Output))

test3Output := execContainer(t, ctr, "cat /test3output")
require.Equal(t, "hello from test 3!", strings.TrimSpace(test3Output))
}

func TestBuildFromDockerfile(t *testing.T) {
Expand Down

0 comments on commit b5e15ad

Please sign in to comment.