From fe17ab4e6e19a7eba1da6e24968f928118694c96 Mon Sep 17 00:00:00 2001 From: Natalie Arellano Date: Tue, 22 Oct 2024 15:20:09 -0400 Subject: [PATCH] Exporter does not re-use layer from volume cache if layer contents are corrupted Signed-off-by: Natalie Arellano --- Makefile | 2 +- acceptance/acceptance_test.go | 1 - acceptance/analyzer_test.go | 1 - acceptance/creator_test.go | 1 - acceptance/exporter_test.go | 691 ++++++++++-------- .../io.buildpacks.lifecycle.cache.metadata | 17 + ...66ebc91139e7c0e574f60d1d4092f53d7dff59.tar | Bin 0 -> 10240 bytes .../corrupted_buildpack/corrupted-layer.toml | 3 + .../corrupted_buildpack/corrupted-layer/data | 1 + .../exporter/container/layers/group.toml | 5 + cache/image_cache.go | 1 + cache/volume_cache.go | 1 + cmd/lifecycle/restorer.go | 12 +- phase/cache.go | 24 +- phase/rebaser.go | 2 +- 15 files changed, 432 insertions(+), 330 deletions(-) create mode 100644 acceptance/testdata/exporter/cache-dir/committed/io.buildpacks.lifecycle.cache.metadata create mode 100644 acceptance/testdata/exporter/cache-dir/committed/sha256:258dfa0cc987efebc17559694866ebc91139e7c0e574f60d1d4092f53d7dff59.tar create mode 100644 acceptance/testdata/exporter/container/layers/corrupted_buildpack/corrupted-layer.toml create mode 100644 acceptance/testdata/exporter/container/layers/corrupted_buildpack/corrupted-layer/data diff --git a/Makefile b/Makefile index 84ebbd882..d819d5179 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,7 @@ install-mockgen: install-golangci-lint: @echo "> Installing golangci-lint..." - $(GOCMD) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.57.2 + $(GOCMD) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61.0 lint: install-golangci-lint @echo "> Linting code..." diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index a1deec531..7b8ba9a19 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -23,7 +23,6 @@ const ( var ( latestPlatformAPI = api.Platform.Latest().String() buildDir string - cacheFixtureDir string ) func TestVersion(t *testing.T) { diff --git a/acceptance/analyzer_test.go b/acceptance/analyzer_test.go index 92e9d487f..23abb9e45 100644 --- a/acceptance/analyzer_test.go +++ b/acceptance/analyzer_test.go @@ -37,7 +37,6 @@ func TestAnalyzer(t *testing.T) { analyzeImage = analyzeTest.testImageRef analyzerPath = analyzeTest.containerBinaryPath - cacheFixtureDir = filepath.Join("testdata", "cache-dir") analyzeRegAuthConfig = analyzeTest.targetRegistry.authConfig analyzeRegNetwork = analyzeTest.targetRegistry.network analyzeDaemonFixtures = analyzeTest.targetDaemon.fixtures diff --git a/acceptance/creator_test.go b/acceptance/creator_test.go index 749ee4e25..ebd38788d 100644 --- a/acceptance/creator_test.go +++ b/acceptance/creator_test.go @@ -40,7 +40,6 @@ func TestCreator(t *testing.T) { createImage = createTest.testImageRef creatorPath = createTest.containerBinaryPath - cacheFixtureDir = filepath.Join("testdata", "creator", "cache-dir") createRegAuthConfig = createTest.targetRegistry.authConfig createRegNetwork = createTest.targetRegistry.network createDaemonFixtures = createTest.targetDaemon.fixtures diff --git a/acceptance/exporter_test.go b/acceptance/exporter_test.go index 493316579..1c45beace 100644 --- a/acceptance/exporter_test.go +++ b/acceptance/exporter_test.go @@ -5,8 +5,11 @@ package acceptance import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -18,6 +21,7 @@ import ( "github.com/buildpacks/imgutil" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/pkg/errors" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -25,6 +29,7 @@ import ( "github.com/buildpacks/lifecycle/auth" "github.com/buildpacks/lifecycle/cache" "github.com/buildpacks/lifecycle/cmd" + "github.com/buildpacks/lifecycle/internal/fsutil" "github.com/buildpacks/lifecycle/internal/path" "github.com/buildpacks/lifecycle/platform/files" h "github.com/buildpacks/lifecycle/testhelpers" @@ -71,140 +76,138 @@ func testExporterFunc(platformAPI string) func(t *testing.T, when spec.G, it spe }) when("daemon case", func() { - when("first build", func() { - when("app", func() { - it("is created", func() { - exportFlags := []string{"-daemon", "-log-level", "debug"} - exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) - exportedImageName = "some-exported-image-" + h.RandString(10) - exportArgs = append(exportArgs, exportedImageName) - - output := h.DockerRun(t, - exportImage, - h.WithFlags(append( - dockerSocketMount, - "--env", "CNB_PLATFORM_API="+platformAPI, - )...), - h.WithArgs(exportArgs...), - ) - h.AssertStringContains(t, output, "Saving "+exportedImageName) + it("app is created", func() { + exportFlags := []string{"-daemon", "-log-level", "debug"} + exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) + exportedImageName = "some-exported-image-" + h.RandString(10) + exportArgs = append(exportArgs, exportedImageName) + + output := h.DockerRun(t, + exportImage, + h.WithFlags(append( + dockerSocketMount, + "--env", "CNB_PLATFORM_API="+platformAPI, + )...), + h.WithArgs(exportArgs...), + ) + h.AssertStringContains(t, output, "Saving "+exportedImageName) + + if api.MustParse(platformAPI).AtLeast("0.11") { + extensions := []string{"sbom.cdx.json", "sbom.spdx.json", "sbom.syft.json"} + for _, extension := range extensions { + h.AssertStringContains(t, output, fmt.Sprintf("Copying SBOM lifecycle.%s to %s", extension, filepath.Join(path.RootDir, "layers", "sbom", "build", "buildpacksio_lifecycle", extension))) + h.AssertStringContains(t, output, fmt.Sprintf("Copying SBOM launcher.%s to %s", extension, filepath.Join(path.RootDir, "layers", "sbom", "launch", "buildpacksio_lifecycle", "launcher", extension))) + } + } else { + h.AssertStringDoesNotContain(t, output, "Copying SBOM") + } + + if api.MustParse(platformAPI).AtLeast("0.12") { + expectedHistory := []string{ + "Buildpacks Launcher Config", + "Buildpacks Application Launcher", + "Application Layer", + "Software Bill-of-Materials", + "Layer: 'corrupted-layer', Created by buildpack: corrupted_buildpack@corrupted_v1", + "Layer: 'launch-layer', Created by buildpack: cacher_buildpack@cacher_v1", + "", // run image layer + } + assertDaemonImageHasHistory(t, exportedImageName, expectedHistory) + } else { + assertDaemonImageDoesNotHaveHistory(t, exportedImageName) + } - if api.MustParse(platformAPI).AtLeast("0.11") { - extensions := []string{"sbom.cdx.json", "sbom.spdx.json", "sbom.syft.json"} - for _, extension := range extensions { - h.AssertStringContains(t, output, fmt.Sprintf("Copying SBOM lifecycle.%s to %s", extension, filepath.Join(path.RootDir, "layers", "sbom", "build", "buildpacksio_lifecycle", extension))) - h.AssertStringContains(t, output, fmt.Sprintf("Copying SBOM launcher.%s to %s", extension, filepath.Join(path.RootDir, "layers", "sbom", "launch", "buildpacksio_lifecycle", "launcher", extension))) - } - } else { - h.AssertStringDoesNotContain(t, output, "Copying SBOM") - } - - if api.MustParse(platformAPI).AtLeast("0.12") { - expectedHistory := []string{ - "Buildpacks Launcher Config", - "Buildpacks Application Launcher", - "Application Layer", - "Software Bill-of-Materials", - "Layer: 'launch-layer', Created by buildpack: cacher_buildpack@cacher_v1", - "", // run image layer - } - assertDaemonImageHasHistory(t, exportedImageName, expectedHistory) - } else { - assertDaemonImageDoesNotHaveHistory(t, exportedImageName) - } + assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) + }) - assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) - }) + when("using extensions", func() { + it.Before(func() { + h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.12"), "") }) - when("using extensions", func() { - it.Before(func() { - h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.12"), "") - }) - - it("is created from the extended run image", func() { - exportFlags := []string{ - "-analyzed", "/layers/run-image-extended-analyzed.toml", // though the run image is a registry image, it also exists in the daemon with the same tag - "-daemon", - "-extended", "/layers/some-extended-dir", - "-log-level", "debug", - "-run", "/cnb/run.toml", // though the run image is a registry image, it also exists in the daemon with the same tag - } - exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) - exportedImageName = "some-exported-image-" + h.RandString(10) - exportArgs = append(exportArgs, exportedImageName) + it("app is created from the extended run image", func() { + exportFlags := []string{ + "-analyzed", "/layers/run-image-extended-analyzed.toml", // though the run image is a registry image, it also exists in the daemon with the same tag + "-daemon", + "-extended", "/layers/some-extended-dir", + "-log-level", "debug", + "-run", "/cnb/run.toml", // though the run image is a registry image, it also exists in the daemon with the same tag + } + exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) + exportedImageName = "some-exported-image-" + h.RandString(10) + exportArgs = append(exportArgs, exportedImageName) - // get run image top layer - inspect, _, err := h.DockerCli(t).ImageInspectWithRaw(context.TODO(), exportTest.targetRegistry.fixtures.ReadOnlyRunImage) - h.AssertNil(t, err) - layers := inspect.RootFS.Layers - runImageFixtureTopLayerSHA := layers[len(layers)-1] - runImageFixtureSHA := inspect.ID + // get run image top layer + inspect, _, err := h.DockerCli(t).ImageInspectWithRaw(context.TODO(), exportTest.targetRegistry.fixtures.ReadOnlyRunImage) + h.AssertNil(t, err) + layers := inspect.RootFS.Layers + runImageFixtureTopLayerSHA := layers[len(layers)-1] + runImageFixtureSHA := inspect.ID - experimentalMode := "warn" - if api.MustParse(platformAPI).AtLeast("0.13") { - experimentalMode = "error" - } + experimentalMode := "warn" + if api.MustParse(platformAPI).AtLeast("0.13") { + experimentalMode = "error" + } - output := h.DockerRun(t, - exportImage, - h.WithFlags(append( - dockerSocketMount, - "--env", "CNB_EXPERIMENTAL_MODE="+experimentalMode, - "--env", "CNB_PLATFORM_API="+platformAPI, - )...), - h.WithArgs(exportArgs...), - ) - h.AssertStringContains(t, output, "Saving "+exportedImageName) + output := h.DockerRun(t, + exportImage, + h.WithFlags(append( + dockerSocketMount, + "--env", "CNB_EXPERIMENTAL_MODE="+experimentalMode, + "--env", "CNB_PLATFORM_API="+platformAPI, + )...), + h.WithArgs(exportArgs...), + ) + h.AssertStringContains(t, output, "Saving "+exportedImageName) - assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) - expectedHistory := []string{ - "Buildpacks Launcher Config", - "Buildpacks Application Launcher", - "Application Layer", - "Software Bill-of-Materials", - "Layer: 'launch-layer', Created by buildpack: cacher_buildpack@cacher_v1", - "Layer: 'RUN mkdir /some-other-dir && echo some-data > /some-other-dir/some-file && echo some-data > /some-other-file', Created by extension: second-extension", - "Layer: 'RUN mkdir /some-dir && echo some-data > /some-dir/some-file && echo some-data > /some-file', Created by extension: first-extension", - "", // run image layer - } - assertDaemonImageHasHistory(t, exportedImageName, expectedHistory) - t.Log("bases the exported image on the extended run image") - inspect, _, err = h.DockerCli(t).ImageInspectWithRaw(context.TODO(), exportedImageName) - h.AssertNil(t, err) - h.AssertEq(t, inspect.Config.Labels["io.buildpacks.rebasable"], "false") // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/ - t.Log("Adds extension layers") - type testCase struct { - expectedDiffID string - layerIndex int - } - testCases := []testCase{ - { - expectedDiffID: "sha256:fb54d2566824d6630d94db0b008d9a544a94d3547a424f52e2fd282b648c0601", // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/65c2873d397056a5cb4169790654d787579b005f18b903082b177d4d9b4aecf5 after un-compressing and zeroing timestamps - layerIndex: 1, - }, - { - expectedDiffID: "sha256:1018c7d3584c4f7fa3ef4486d1a6a11b93956b9d8bfe0898a3e0fbd248c984d8", // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/0fb9b88c9cbe9f11b4c8da645f390df59f5949632985a0bfc2a842ef17b2ad18 after un-compressing and zeroing timestamps - layerIndex: 2, - }, - } - for _, tc := range testCases { - h.AssertEq(t, inspect.RootFS.Layers[tc.layerIndex], tc.expectedDiffID) - } - t.Log("sets the layers metadata label according to the new spec") - var lmd files.LayersMetadata - lmdJSON := inspect.Config.Labels["io.buildpacks.lifecycle.metadata"] - h.AssertNil(t, json.Unmarshal([]byte(lmdJSON), &lmd)) - h.AssertEq(t, lmd.RunImage.Image, exportTest.targetRegistry.fixtures.ReadOnlyRunImage) // from analyzed.toml - h.AssertEq(t, lmd.RunImage.Mirrors, []string{"mirror1", "mirror2"}) // from run.toml - h.AssertEq(t, lmd.RunImage.TopLayer, runImageFixtureTopLayerSHA) - h.AssertEq(t, lmd.RunImage.Reference, strings.TrimPrefix(runImageFixtureSHA, "sha256:")) - }) + assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) + expectedHistory := []string{ + "Buildpacks Launcher Config", + "Buildpacks Application Launcher", + "Application Layer", + "Software Bill-of-Materials", + "Layer: 'corrupted-layer', Created by buildpack: corrupted_buildpack@corrupted_v1", + "Layer: 'launch-layer', Created by buildpack: cacher_buildpack@cacher_v1", + "Layer: 'RUN mkdir /some-other-dir && echo some-data > /some-other-dir/some-file && echo some-data > /some-other-file', Created by extension: second-extension", + "Layer: 'RUN mkdir /some-dir && echo some-data > /some-dir/some-file && echo some-data > /some-file', Created by extension: first-extension", + "", // run image layer + } + assertDaemonImageHasHistory(t, exportedImageName, expectedHistory) + t.Log("bases the exported image on the extended run image") + inspect, _, err = h.DockerCli(t).ImageInspectWithRaw(context.TODO(), exportedImageName) + h.AssertNil(t, err) + h.AssertEq(t, inspect.Config.Labels["io.buildpacks.rebasable"], "false") // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/ + t.Log("Adds extension layers") + type testCase struct { + expectedDiffID string + layerIndex int + } + testCases := []testCase{ + { + expectedDiffID: "sha256:fb54d2566824d6630d94db0b008d9a544a94d3547a424f52e2fd282b648c0601", // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/65c2873d397056a5cb4169790654d787579b005f18b903082b177d4d9b4aecf5 after un-compressing and zeroing timestamps + layerIndex: 1, + }, + { + expectedDiffID: "sha256:1018c7d3584c4f7fa3ef4486d1a6a11b93956b9d8bfe0898a3e0fbd248c984d8", // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/0fb9b88c9cbe9f11b4c8da645f390df59f5949632985a0bfc2a842ef17b2ad18 after un-compressing and zeroing timestamps + layerIndex: 2, + }, + } + for _, tc := range testCases { + h.AssertEq(t, inspect.RootFS.Layers[tc.layerIndex], tc.expectedDiffID) + } + t.Log("sets the layers metadata label according to the new spec") + var lmd files.LayersMetadata + lmdJSON := inspect.Config.Labels["io.buildpacks.lifecycle.metadata"] + h.AssertNil(t, json.Unmarshal([]byte(lmdJSON), &lmd)) + h.AssertEq(t, lmd.RunImage.Image, exportTest.targetRegistry.fixtures.ReadOnlyRunImage) // from analyzed.toml + h.AssertEq(t, lmd.RunImage.Mirrors, []string{"mirror1", "mirror2"}) // from run.toml + h.AssertEq(t, lmd.RunImage.TopLayer, runImageFixtureTopLayerSHA) + h.AssertEq(t, lmd.RunImage.Reference, strings.TrimPrefix(runImageFixtureSHA, "sha256:")) }) }) when("SOURCE_DATE_EPOCH is set", func() { - it("Image CreatedAt is set to SOURCE_DATE_EPOCH", func() { + it("app is created with config CreatedAt set to SOURCE_DATE_EPOCH", func() { h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.9"), "SOURCE_DATE_EPOCH support added in 0.9") expectedTime := time.Date(2022, 1, 5, 5, 5, 5, 0, time.UTC) exportFlags := []string{"-daemon"} @@ -231,63 +234,87 @@ func testExporterFunc(platformAPI string) func(t *testing.T, when spec.G, it spe }) when("registry case", func() { - when("first build", func() { - when("app", func() { - it("is created", func() { - var exportFlags []string - exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) - exportedImageName = exportTest.RegRepoName("some-exported-image-" + h.RandString(10)) - exportArgs = append(exportArgs, exportedImageName) + it("app is created", func() { + var exportFlags []string + exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) + exportedImageName = exportTest.RegRepoName("some-exported-image-" + h.RandString(10)) + exportArgs = append(exportArgs, exportedImageName) + + output := h.DockerRun(t, + exportImage, + h.WithFlags( + "--env", "CNB_PLATFORM_API="+platformAPI, + "--env", "CNB_REGISTRY_AUTH="+exportRegAuthConfig, + "--network", exportRegNetwork, + ), + h.WithArgs(exportArgs...), + ) + h.AssertStringContains(t, output, "Saving "+exportedImageName) + + h.Run(t, exec.Command("docker", "pull", exportedImageName)) + assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) + }) - output := h.DockerRun(t, - exportImage, - h.WithFlags( - "--env", "CNB_PLATFORM_API="+platformAPI, - "--env", "CNB_REGISTRY_AUTH="+exportRegAuthConfig, - "--network", exportRegNetwork, - ), - h.WithArgs(exportArgs...), - ) - h.AssertStringContains(t, output, "Saving "+exportedImageName) + when("registry is insecure", func() { + it.Before(func() { + h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.12"), "") + }) - h.Run(t, exec.Command("docker", "pull", exportedImageName)) - assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) - }) + it("uses http protocol", func() { + var exportFlags []string + exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) + exportedImageName = exportTest.RegRepoName("some-insecure-exported-image-" + h.RandString(10)) + exportArgs = append(exportArgs, exportedImageName) + insecureRegistry := "host.docker.internal/bar" + insecureAnalyzed := "/layers/analyzed_insecure.toml" + + _, _, err := h.DockerRunWithError(t, + exportImage, + h.WithFlags( + "--env", "CNB_PLATFORM_API="+platformAPI, + "--env", "CNB_INSECURE_REGISTRIES="+insecureRegistry, + "--env", "CNB_ANALYZED_PATH="+insecureAnalyzed, + "--network", exportRegNetwork, + ), + h.WithArgs(exportArgs...), + ) + h.AssertStringContains(t, err.Error(), "http://host.docker.internal") }) + }) - when("app using insecure registry", func() { - it.Before(func() { - h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.12"), "") - }) + when("SOURCE_DATE_EPOCH is set", func() { + it("app is created with config CreatedAt set to SOURCE_DATE_EPOCH", func() { + h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.9"), "SOURCE_DATE_EPOCH support added in 0.9") + expectedTime := time.Date(2022, 1, 5, 5, 5, 5, 0, time.UTC) - it("does an http request", func() { - var exportFlags []string - exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) - exportedImageName = exportTest.RegRepoName("some-insecure-exported-image-" + h.RandString(10)) - exportArgs = append(exportArgs, exportedImageName) - insecureRegistry := "host.docker.internal/bar" - insecureAnalyzed := "/layers/analyzed_insecure.toml" + var exportFlags []string + exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) + exportedImageName = exportTest.RegRepoName("some-exported-image-" + h.RandString(10)) + exportArgs = append(exportArgs, exportedImageName) - _, _, err := h.DockerRunWithError(t, - exportImage, - h.WithFlags( - "--env", "CNB_PLATFORM_API="+platformAPI, - "--env", "CNB_INSECURE_REGISTRIES="+insecureRegistry, - "--env", "CNB_ANALYZED_PATH="+insecureAnalyzed, - "--network", exportRegNetwork, - ), - h.WithArgs(exportArgs...), - ) - h.AssertStringContains(t, err.Error(), "http://host.docker.internal") - }) - }) + output := h.DockerRun(t, + exportImage, + h.WithFlags( + "--env", "CNB_PLATFORM_API="+platformAPI, + "--env", "CNB_REGISTRY_AUTH="+exportRegAuthConfig, + "--env", "SOURCE_DATE_EPOCH="+fmt.Sprintf("%d", expectedTime.Unix()), + "--network", exportRegNetwork, + ), + h.WithArgs(exportArgs...), + ) + h.AssertStringContains(t, output, "Saving "+exportedImageName) - when("SOURCE_DATE_EPOCH is set", func() { - it("Image CreatedAt is set to SOURCE_DATE_EPOCH", func() { - h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.9"), "SOURCE_DATE_EPOCH support added in 0.9") - expectedTime := time.Date(2022, 1, 5, 5, 5, 5, 0, time.UTC) + h.Run(t, exec.Command("docker", "pull", exportedImageName)) + assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, expectedTime) + }) + }) - var exportFlags []string + // FIXME: move this out of the registry block + when("cache", func() { + when("image case", func() { + it("cache is created", func() { + cacheImageName := exportTest.RegRepoName("some-cache-image-" + h.RandString(10)) + exportFlags := []string{"-cache-image", cacheImageName} exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) exportedImageName = exportTest.RegRepoName("some-exported-image-" + h.RandString(10)) exportArgs = append(exportArgs, exportedImageName) @@ -297,23 +324,21 @@ func testExporterFunc(platformAPI string) func(t *testing.T, when spec.G, it spe h.WithFlags( "--env", "CNB_PLATFORM_API="+platformAPI, "--env", "CNB_REGISTRY_AUTH="+exportRegAuthConfig, - "--env", "SOURCE_DATE_EPOCH="+fmt.Sprintf("%d", expectedTime.Unix()), "--network", exportRegNetwork, ), h.WithArgs(exportArgs...), ) h.AssertStringContains(t, output, "Saving "+exportedImageName) - + // To detect whether the export of cacheImage and exportedImage is successful h.Run(t, exec.Command("docker", "pull", exportedImageName)) - assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, expectedTime) + assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) + h.Run(t, exec.Command("docker", "pull", cacheImageName)) }) - }) - when("cache", func() { - when("cache image case", func() { - it("is created", func() { + when("parallel export is enabled", func() { + it("cache is created", func() { cacheImageName := exportTest.RegRepoName("some-cache-image-" + h.RandString(10)) - exportFlags := []string{"-cache-image", cacheImageName} + exportFlags := []string{"-cache-image", cacheImageName, "-parallel"} exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) exportedImageName = exportTest.RegRepoName("some-exported-image-" + h.RandString(10)) exportArgs = append(exportArgs, exportedImageName) @@ -328,15 +353,17 @@ func testExporterFunc(platformAPI string) func(t *testing.T, when spec.G, it spe h.WithArgs(exportArgs...), ) h.AssertStringContains(t, output, "Saving "+exportedImageName) - // To detect whether the export of cacheImage and exportedImage is successful + h.Run(t, exec.Command("docker", "pull", exportedImageName)) assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) h.Run(t, exec.Command("docker", "pull", cacheImageName)) }) + }) - it("is created with parallel export enabled", func() { - cacheImageName := exportTest.RegRepoName("some-cache-image-" + h.RandString(10)) - exportFlags := []string{"-cache-image", cacheImageName, "-parallel"} + when("cache is provided but no data was cached", func() { + it("cache is created with an empty layer", func() { + cacheImageName := exportTest.RegRepoName("some-empty-cache-image-" + h.RandString(10)) + exportFlags := []string{"-cache-image", cacheImageName, "-layers", "/other_layers"} exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) exportedImageName = exportTest.RegRepoName("some-exported-image-" + h.RandString(10)) exportArgs = append(exportArgs, exportedImageName) @@ -352,14 +379,45 @@ func testExporterFunc(platformAPI string) func(t *testing.T, when spec.G, it spe ) h.AssertStringContains(t, output, "Saving "+exportedImageName) - h.Run(t, exec.Command("docker", "pull", exportedImageName)) - assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) + testEmptyLayerSHA := calculateEmptyLayerSha(t) + + // Retrieve the cache image from the ephemeral registry h.Run(t, exec.Command("docker", "pull", cacheImageName)) + logger := cmd.DefaultLogger + + subject, err := cache.NewImageCacheFromName(cacheImageName, authn.DefaultKeychain, logger, cache.NewImageDeleter(cache.NewImageComparer(), logger, api.MustParse(platformAPI).LessThan("0.13"))) + h.AssertNil(t, err) + + //Assert the cache image was created with an empty layer + layer, err := subject.RetrieveLayer(testEmptyLayerSHA) + h.AssertNil(t, err) + defer layer.Close() }) + }) + }) - it("is created with empty layer", func() { - cacheImageName := exportTest.RegRepoName("some-empty-cache-image-" + h.RandString(10)) - exportFlags := []string{"-cache-image", cacheImageName, "-layers", "/other_layers"} + when("directory case", func() { + when("original cache was corrupted", func() { + var cacheDir string + + it.Before(func() { + var err error + cacheDir, err = os.MkdirTemp("", "cache") + h.AssertNil(t, err) + + cacheFixtureDir := filepath.Join("testdata", "exporter", "cache-dir") + h.AssertNil(t, fsutil.Copy(cacheFixtureDir, cacheDir)) + }) + + it.After(func() { + _ = os.RemoveAll(cacheDir) + }) + + it("overwrites the original layer", func() { + exportFlags := []string{ + "-cache-dir", "/cache/committed", + "-log-level", "debug", + } exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) exportedImageName = exportTest.RegRepoName("some-exported-image-" + h.RandString(10)) exportArgs = append(exportArgs, exportedImageName) @@ -370,115 +428,130 @@ func testExporterFunc(platformAPI string) func(t *testing.T, when spec.G, it spe "--env", "CNB_PLATFORM_API="+platformAPI, "--env", "CNB_REGISTRY_AUTH="+exportRegAuthConfig, "--network", exportRegNetwork, + "--volume", fmt.Sprintf("%s:/cache/committed", cacheDir), ), h.WithArgs(exportArgs...), ) h.AssertStringContains(t, output, "Saving "+exportedImageName) - - testEmptyLayerSHA := calculateEmptyLayerSha(t) - - // Retrieve the cache image from the ephemeral registry - h.Run(t, exec.Command("docker", "pull", cacheImageName)) - logger := cmd.DefaultLogger - - subject, err := cache.NewImageCacheFromName(cacheImageName, authn.DefaultKeychain, logger, cache.NewImageDeleter(cache.NewImageComparer(), logger, api.MustParse(platformAPI).LessThan("0.13"))) + h.Run(t, exec.Command("docker", "pull", exportedImageName)) + defer h.Run(t, exec.Command("docker", "image", "rm", exportedImageName)) + // Verify the app has the correct sha for the layer + inspect, _, err := h.DockerCli(t).ImageInspectWithRaw(context.TODO(), exportedImageName) h.AssertNil(t, err) - - //Assert the cache image was created with an empty layer - layer, err := subject.RetrieveLayer(testEmptyLayerSHA) + var lmd files.LayersMetadata + lmdJSON := inspect.Config.Labels["io.buildpacks.lifecycle.metadata"] + h.AssertNil(t, json.Unmarshal([]byte(lmdJSON), &lmd)) + h.AssertEq(t, lmd.Buildpacks[2].Layers["corrupted-layer"].SHA, "sha256:258dfa0cc987efebc17559694866ebc91139e7c0e574f60d1d4092f53d7dff59") + // Verify the cache has correct contents now + foundDiffID, err := func() (string, error) { + layerPath := filepath.Join(cacheDir, "committed", "sha256:258dfa0cc987efebc17559694866ebc91139e7c0e574f60d1d4092f53d7dff59.tar") + layerRC, err := os.Open(layerPath) + if err != nil { + return "", err + } + defer func() { + _ = layerRC.Close() + }() + hasher := sha256.New() + if _, err = io.Copy(hasher, layerRC); err != nil { + return "", errors.Wrap(err, "hashing layer") + } + foundDiffID := "sha256:" + hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))) + return foundDiffID, nil + }() h.AssertNil(t, err) - defer layer.Close() + h.AssertEq(t, foundDiffID, "sha256:258dfa0cc987efebc17559694866ebc91139e7c0e574f60d1d4092f53d7dff59") }) }) }) + }) - when("using extensions", func() { - it.Before(func() { - h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.12"), "") - }) + when("using extensions", func() { + it.Before(func() { + h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.12"), "") + }) - it("is created from the extended run image", func() { - exportFlags := []string{ - "-analyzed", "/layers/run-image-extended-analyzed.toml", - "-extended", "/layers/some-extended-dir", - "-log-level", "debug", - "-run", "/cnb/run.toml", - } - exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) - exportedImageName = exportTest.RegRepoName("some-exported-image-" + h.RandString(10)) - exportArgs = append(exportArgs, exportedImageName) + it("app is created from the extended run image", func() { + exportFlags := []string{ + "-analyzed", "/layers/run-image-extended-analyzed.toml", + "-extended", "/layers/some-extended-dir", + "-log-level", "debug", + "-run", "/cnb/run.toml", + } + exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) + exportedImageName = exportTest.RegRepoName("some-exported-image-" + h.RandString(10)) + exportArgs = append(exportArgs, exportedImageName) - // get run image SHA & top layer - ref, imageAuth, err := auth.ReferenceForRepoName(authn.DefaultKeychain, exportTest.targetRegistry.fixtures.ReadOnlyRunImage) - h.AssertNil(t, err) - remoteImage, err := remote.Image(ref, remote.WithAuth(imageAuth)) - h.AssertNil(t, err) - layers, err := remoteImage.Layers() - h.AssertNil(t, err) - runImageFixtureTopLayerSHA, err := layers[len(layers)-1].DiffID() - h.AssertNil(t, err) - runImageFixtureSHA, err := remoteImage.Digest() - h.AssertNil(t, err) + // get run image SHA & top layer + ref, imageAuth, err := auth.ReferenceForRepoName(authn.DefaultKeychain, exportTest.targetRegistry.fixtures.ReadOnlyRunImage) + h.AssertNil(t, err) + remoteImage, err := remote.Image(ref, remote.WithAuth(imageAuth)) + h.AssertNil(t, err) + layers, err := remoteImage.Layers() + h.AssertNil(t, err) + runImageFixtureTopLayerSHA, err := layers[len(layers)-1].DiffID() + h.AssertNil(t, err) + runImageFixtureSHA, err := remoteImage.Digest() + h.AssertNil(t, err) - experimentalMode := "warn" - if api.MustParse(platformAPI).AtLeast("0.13") { - experimentalMode = "error" - } + experimentalMode := "warn" + if api.MustParse(platformAPI).AtLeast("0.13") { + experimentalMode = "error" + } - output := h.DockerRun(t, - exportImage, - h.WithFlags( - "--env", "CNB_EXPERIMENTAL_MODE="+experimentalMode, - "--env", "CNB_PLATFORM_API="+platformAPI, - "--env", "CNB_REGISTRY_AUTH="+exportRegAuthConfig, - "--network", exportRegNetwork, - ), - h.WithArgs(exportArgs...), - ) - h.AssertStringContains(t, output, "Saving "+exportedImageName) + output := h.DockerRun(t, + exportImage, + h.WithFlags( + "--env", "CNB_EXPERIMENTAL_MODE="+experimentalMode, + "--env", "CNB_PLATFORM_API="+platformAPI, + "--env", "CNB_REGISTRY_AUTH="+exportRegAuthConfig, + "--network", exportRegNetwork, + ), + h.WithArgs(exportArgs...), + ) + h.AssertStringContains(t, output, "Saving "+exportedImageName) - h.Run(t, exec.Command("docker", "pull", exportedImageName)) - assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) - t.Log("bases the exported image on the extended run image") - ref, imageAuth, err = auth.ReferenceForRepoName(authn.DefaultKeychain, exportedImageName) - h.AssertNil(t, err) - remoteImage, err = remote.Image(ref, remote.WithAuth(imageAuth)) - h.AssertNil(t, err) - configFile, err := remoteImage.ConfigFile() - h.AssertNil(t, err) - h.AssertEq(t, configFile.Config.Labels["io.buildpacks.rebasable"], "false") // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/ - t.Log("Adds extension layers") - layers, err = remoteImage.Layers() + h.Run(t, exec.Command("docker", "pull", exportedImageName)) + assertImageOSAndArchAndCreatedAt(t, exportedImageName, exportTest, imgutil.NormalizedDateTime) + t.Log("bases the exported image on the extended run image") + ref, imageAuth, err = auth.ReferenceForRepoName(authn.DefaultKeychain, exportedImageName) + h.AssertNil(t, err) + remoteImage, err = remote.Image(ref, remote.WithAuth(imageAuth)) + h.AssertNil(t, err) + configFile, err := remoteImage.ConfigFile() + h.AssertNil(t, err) + h.AssertEq(t, configFile.Config.Labels["io.buildpacks.rebasable"], "false") // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/ + t.Log("Adds extension layers") + layers, err = remoteImage.Layers() + h.AssertNil(t, err) + type testCase struct { + expectedDigest string + layerIndex int + } + testCases := []testCase{ + { + expectedDigest: "sha256:08e7ad5ce17cf5e5f70affe68b341a93de86ee2ba074932c3a05b8770f66d772", // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/65c2873d397056a5cb4169790654d787579b005f18b903082b177d4d9b4aecf5 after un-compressing, zeroing timestamps, and re-compressing + layerIndex: 1, + }, + { + expectedDigest: "sha256:0e74ef444ea437147e3fa0ce2aad371df5380c26b96875ae07b9b67f44cdb2ee", // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/0fb9b88c9cbe9f11b4c8da645f390df59f5949632985a0bfc2a842ef17b2ad18 after un-compressing, zeroing timestamps, and re-compressing + layerIndex: 2, + }, + } + for _, tc := range testCases { + layer := layers[tc.layerIndex] + digest, err := layer.Digest() h.AssertNil(t, err) - type testCase struct { - expectedDigest string - layerIndex int - } - testCases := []testCase{ - { - expectedDigest: "sha256:08e7ad5ce17cf5e5f70affe68b341a93de86ee2ba074932c3a05b8770f66d772", // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/65c2873d397056a5cb4169790654d787579b005f18b903082b177d4d9b4aecf5 after un-compressing, zeroing timestamps, and re-compressing - layerIndex: 1, - }, - { - expectedDigest: "sha256:0e74ef444ea437147e3fa0ce2aad371df5380c26b96875ae07b9b67f44cdb2ee", // from testdata/exporter/container/layers/some-extended-dir/run/sha256_/blobs/sha256/0fb9b88c9cbe9f11b4c8da645f390df59f5949632985a0bfc2a842ef17b2ad18 after un-compressing, zeroing timestamps, and re-compressing - layerIndex: 2, - }, - } - for _, tc := range testCases { - layer := layers[tc.layerIndex] - digest, err := layer.Digest() - h.AssertNil(t, err) - h.AssertEq(t, digest.String(), tc.expectedDigest) - } - t.Log("sets the layers metadata label according to the new spec") - var lmd files.LayersMetadata - lmdJSON := configFile.Config.Labels["io.buildpacks.lifecycle.metadata"] - h.AssertNil(t, json.Unmarshal([]byte(lmdJSON), &lmd)) - h.AssertEq(t, lmd.RunImage.Image, exportTest.targetRegistry.fixtures.ReadOnlyRunImage) // from analyzed.toml - h.AssertEq(t, lmd.RunImage.Mirrors, []string{"mirror1", "mirror2"}) // from run.toml - h.AssertEq(t, lmd.RunImage.TopLayer, runImageFixtureTopLayerSHA.String()) - h.AssertEq(t, lmd.RunImage.Reference, fmt.Sprintf("%s@%s", exportTest.targetRegistry.fixtures.ReadOnlyRunImage, runImageFixtureSHA.String())) - }) + h.AssertEq(t, digest.String(), tc.expectedDigest) + } + t.Log("sets the layers metadata label according to the new spec") + var lmd files.LayersMetadata + lmdJSON := configFile.Config.Labels["io.buildpacks.lifecycle.metadata"] + h.AssertNil(t, json.Unmarshal([]byte(lmdJSON), &lmd)) + h.AssertEq(t, lmd.RunImage.Image, exportTest.targetRegistry.fixtures.ReadOnlyRunImage) // from analyzed.toml + h.AssertEq(t, lmd.RunImage.Mirrors, []string{"mirror1", "mirror2"}) // from run.toml + h.AssertEq(t, lmd.RunImage.TopLayer, runImageFixtureTopLayerSHA.String()) + h.AssertEq(t, lmd.RunImage.Reference, fmt.Sprintf("%s@%s", exportTest.targetRegistry.fixtures.ReadOnlyRunImage, runImageFixtureSHA.String())) }) }) }) @@ -493,7 +566,7 @@ func testExporterFunc(platformAPI string) func(t *testing.T, when spec.G, it spe when("experimental mode is enabled", func() { it.Before(func() { - // creates the directory to save all the OCI images on disk + // create the directory to save all OCI images on disk tmpDir, err = os.MkdirTemp("", "layout") h.AssertNil(t, err) @@ -508,35 +581,31 @@ func testExporterFunc(platformAPI string) func(t *testing.T, when spec.G, it spe os.RemoveAll(tmpDir) }) - when("custom layout directory", func() { - when("first build", func() { - when("app", func() { - it.Before(func() { - exportedImageName = "my-custom-layout-app" - layoutDir = filepath.Join(path.RootDir, "my-layout-dir") - }) - - it("is created", func() { - var exportFlags []string - h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.12"), "Platform API < 0.12 does not accept a -layout flag") - exportFlags = append(exportFlags, []string{"-layout", "-layout-dir", layoutDir, "-analyzed", "/layers/layout-analyzed.toml"}...) - exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) - exportArgs = append(exportArgs, exportedImageName) - - output := h.DockerRunAndCopy(t, containerName, tmpDir, layoutDir, exportImage, - h.WithFlags( - "--env", "CNB_EXPERIMENTAL_MODE=warn", - "--env", "CNB_PLATFORM_API="+platformAPI, - ), - h.WithArgs(exportArgs...)) - - h.AssertStringContains(t, output, "Saving /my-layout-dir/index.docker.io/library/my-custom-layout-app/latest") - - // assert the image was saved on disk in OCI layout format - index := h.ReadIndexManifest(t, filepath.Join(tmpDir, layoutDir, "index.docker.io", "library", exportedImageName, "latest")) - h.AssertEq(t, len(index.Manifests), 1) - }) - }) + when("using a custom layout directory", func() { + it.Before(func() { + exportedImageName = "my-custom-layout-app" + layoutDir = filepath.Join(path.RootDir, "my-layout-dir") + }) + + it("app is created", func() { + var exportFlags []string + h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.12"), "Platform API < 0.12 does not accept a -layout flag") + exportFlags = append(exportFlags, []string{"-layout", "-layout-dir", layoutDir, "-analyzed", "/layers/layout-analyzed.toml"}...) + exportArgs := append([]string{ctrPath(exporterPath)}, exportFlags...) + exportArgs = append(exportArgs, exportedImageName) + + output := h.DockerRunAndCopy(t, containerName, tmpDir, layoutDir, exportImage, + h.WithFlags( + "--env", "CNB_EXPERIMENTAL_MODE=warn", + "--env", "CNB_PLATFORM_API="+platformAPI, + ), + h.WithArgs(exportArgs...)) + + h.AssertStringContains(t, output, "Saving /my-layout-dir/index.docker.io/library/my-custom-layout-app/latest") + + // assert the image was saved on disk in OCI layout format + index := h.ReadIndexManifest(t, filepath.Join(tmpDir, layoutDir, "index.docker.io", "library", exportedImageName, "latest")) + h.AssertEq(t, len(index.Manifests), 1) }) }) }) diff --git a/acceptance/testdata/exporter/cache-dir/committed/io.buildpacks.lifecycle.cache.metadata b/acceptance/testdata/exporter/cache-dir/committed/io.buildpacks.lifecycle.cache.metadata new file mode 100644 index 000000000..7046df52d --- /dev/null +++ b/acceptance/testdata/exporter/cache-dir/committed/io.buildpacks.lifecycle.cache.metadata @@ -0,0 +1,17 @@ +{ + "buildpacks": [ + { + "key": "corrupted_buildpack", + "version": "corrupted_v1", + "layers": { + "corrupted-layer": { + "sha": "sha256:258dfa0cc987efebc17559694866ebc91139e7c0e574f60d1d4092f53d7dff59", + "data": null, + "build": false, + "launch": true, + "cache": true + } + } + } + ] +} diff --git a/acceptance/testdata/exporter/cache-dir/committed/sha256:258dfa0cc987efebc17559694866ebc91139e7c0e574f60d1d4092f53d7dff59.tar b/acceptance/testdata/exporter/cache-dir/committed/sha256:258dfa0cc987efebc17559694866ebc91139e7c0e574f60d1d4092f53d7dff59.tar new file mode 100644 index 0000000000000000000000000000000000000000..91ae6ae71dab4a1bd84f58bc8d342d4f6141b342 GIT binary patch literal 10240 zcmeIy-)@316vy#i`xLwY*=eQpah48h3Nid-`+Ho7MjoR