From f1b28be90d5aedcfaa5e19ef23acc58f7366e092 Mon Sep 17 00:00:00 2001 From: Jens Arnfast Date: Mon, 13 May 2024 21:18:24 +0200 Subject: [PATCH 1/2] Addition of experimental flag `full-control-dockerfile` When used porter will use the file referenced by `dockerfile` without any changes (templating, `CMD`, et al) Can be used for building truly custom invocation images. E.g.: - multistage builds - minimizing CVEs - "flatten" image to improve archive performance Signed-off-by: Jens Arnfast --- .../docs/configuration/configuration.md | 10 ++ .../intro-configuration.md | 10 ++ pkg/build/dockerfile-generator.go | 46 +++++++- pkg/build/dockerfile-generator_test.go | 111 ++++++++++++++++++ pkg/experimental/experimental.go | 8 ++ 5 files changed, 182 insertions(+), 3 deletions(-) diff --git a/docs/content/docs/configuration/configuration.md b/docs/content/docs/configuration/configuration.md index 44c8a44d3..b7d3dfa6d 100644 --- a/docs/content/docs/configuration/configuration.md +++ b/docs/content/docs/configuration/configuration.md @@ -22,6 +22,7 @@ You may set a default value for a configuration value in the config file, overri - [Build Drivers](#build-drivers) - [Structured Logs](#structured-logs) - [Dependencies v2](#dependencies-v2) + - [Full control Dockerfile](#full-control-dockerfile) - [Common Configuration Settings](#common-configuration-settings) - [Set Current Namespace](#namespace) - [Output Formatting](#output) @@ -278,6 +279,15 @@ telemetry: The `dependencies-v2` experimental flag is not yet implemented. When it is completed, it is used to activate the features from [PEP003 - Advanced Dependencies](https://github.com/getporter/proposals/blob/main/pep/003-advanced-dependencies.md). +### Full control Dockerfile + +The `full-control-dockerfile` experimental flag disables all Dockerfile generation when building bundles. +When enabled Porter will use the file referenced by `dockerfile` when building the invocation image without modifying it in any way. +This includes injection of any `# PORTER_x` templates, user configuration and `CMD` statements. +It is up to the bundle author to ensure that the file referenced by `dockerfile` contains the necessary tools for any mixins to function and a layout that can be executed as a Porter bundle. + +Note that `autobuild` does not detect changes to the contents of the file referenced by `dockerfile`. + ## Common Configuration Settings Some configuration settings are applicable to many of Porter's commands and to save time you may want to set these values in the configuration file or with environment variables. diff --git a/docs/content/docs/introduction/concepts-and-components/intro-configuration.md b/docs/content/docs/introduction/concepts-and-components/intro-configuration.md index d62be30e3..f7236c33d 100644 --- a/docs/content/docs/introduction/concepts-and-components/intro-configuration.md +++ b/docs/content/docs/introduction/concepts-and-components/intro-configuration.md @@ -19,6 +19,7 @@ You may set a default value for a configuration value in the config file, overri - [Build Drivers](#build-drivers) - [Structured Logs](#structured-logs) - [Dependencies v2](#dependencies-v2) + - [Full control Dockerfile](#full-control-dockerfile) - [Common Configuration Settings](#common-configuration-settings) - [Set Current Namespace](#namespace) - [Output Formatting](#output) @@ -275,6 +276,15 @@ telemetry: The `dependencies-v2` experimental flag is not yet implemented. When it is completed, it is used to activate the features from [PEP003 - Advanced Dependencies](https://github.com/getporter/proposals/blob/main/pep/003-dependency-namespaces-and-labels.md). +### Full control Dockerfile + +The `full-control-dockerfile` experimental flag disables all Dockerfile generation when building bundles. +When enabled Porter will use the file referenced by `dockerfile` when building the invocation image without modifying it in any way. +This includes injection of any `# PORTER_x` templates, user configuration and `CMD` statements. +It is up to the bundle author to ensure that the file referenced by `dockerfile` contains the necessary tools for any mixins to function and a layout that can be executed as a Porter bundle. + +Note that `autobuild` does not detect changes to the contents of the file referenced by `dockerfile`. + ## Common Configuration Settings Some configuration settings are applicable to many of Porter's commands and to save time you may want to set these values in the configuration file or with environment variables. diff --git a/pkg/build/dockerfile-generator.go b/pkg/build/dockerfile-generator.go index 688c2a3f3..1922f7d57 100644 --- a/pkg/build/dockerfile-generator.go +++ b/pkg/build/dockerfile-generator.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "errors" "fmt" "io" "path/filepath" @@ -11,6 +12,7 @@ import ( "get.porter.sh/porter/pkg" "get.porter.sh/porter/pkg/config" + "get.porter.sh/porter/pkg/experimental" "get.porter.sh/porter/pkg/manifest" "get.porter.sh/porter/pkg/mixin/query" "get.porter.sh/porter/pkg/pkgmgmt" @@ -45,9 +47,20 @@ func (g *DockerfileGenerator) GenerateDockerFile(ctx context.Context) error { ctx, span := tracing.StartSpan(ctx) defer span.EndSpan() - lines, err := g.buildDockerfile(ctx) - if err != nil { - return span.Error(fmt.Errorf("error generating the Dockerfile: %w", err)) + var lines []string + var err error + if g.Config.IsFeatureEnabled(experimental.FlagFullControlDockerfile) { + span.Warnf("Experimental feature \"%s\" enabled: Dockerfile will be used without changes by Porter", + experimental.FullControlDockerfile) + lines, err = g.readRawDockerfile(ctx) + if err != nil { + return span.Error(fmt.Errorf("error reading the Dockerfile: %w", err)) + } + } else { + lines, err = g.buildDockerfile(ctx) + if err != nil { + return span.Error(fmt.Errorf("error generating the Dockerfile: %w", err)) + } } contents := strings.Join(lines, "\n") @@ -63,6 +76,33 @@ func (g *DockerfileGenerator) GenerateDockerFile(ctx context.Context) error { return nil } +func (g *DockerfileGenerator) readRawDockerfile(ctx context.Context) ([]string, error) { + if g.Manifest.Dockerfile == "" { + return nil, errors.New("no Dockerfile specified in the manifest") + } + exists, err := g.FileSystem.Exists(g.Manifest.Dockerfile) + if err != nil { + return nil, fmt.Errorf("error checking if Dockerfile exists: %q: %w", g.Manifest.Dockerfile, err) + } + if !exists { + return nil, fmt.Errorf("the Dockerfile specified in the manifest doesn't exist: %q", g.Manifest.Dockerfile) + } + + file, err := g.FileSystem.Open(g.Manifest.Dockerfile) + if err != nil { + return nil, err + } + defer file.Close() + + var lines []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + return lines, scanner.Err() +} + func (g *DockerfileGenerator) buildDockerfile(ctx context.Context) ([]string, error) { log := tracing.LoggerFromContext(ctx) log.Debug("Generating Dockerfile") diff --git a/pkg/build/dockerfile-generator_test.go b/pkg/build/dockerfile-generator_test.go index 54cdbcf26..97e287757 100644 --- a/pkg/build/dockerfile-generator_test.go +++ b/pkg/build/dockerfile-generator_test.go @@ -7,6 +7,7 @@ import ( "testing" "get.porter.sh/porter/pkg/config" + "get.porter.sh/porter/pkg/experimental" "get.porter.sh/porter/pkg/manifest" "get.porter.sh/porter/pkg/mixin" "get.porter.sh/porter/pkg/templates" @@ -255,3 +256,113 @@ func TestPorter_buildMixinsSection_mixinErr(t *testing.T) { _, err = g.buildMixinsSection(ctx) require.EqualError(t, err, "1 error occurred:\n\t* error encountered from mixin \"exec\": encountered build error\n\n") } + +func TestPorter_GenerateDockerfile_WithExperimentalFlagFullControlDockerfile(t *testing.T) { + t.Parallel() + + ctx := context.Background() + c := config.NewTestConfig(t) + + // Enable the experimental feature + c.SetExperimentalFlags(experimental.FlagFullControlDockerfile) + + defer c.Close() + + // Start a span so we can capture the output + ctx, log := c.StartRootSpan(ctx, t.Name()) + defer log.Close() + + tmpl := templates.NewTemplates(c.Config) + configTpl, err := tmpl.GetManifest() + require.Nil(t, err) + c.TestContext.AddTestFileContents(configTpl, config.Name) + + m, err := manifest.LoadManifestFrom(ctx, c.Config, config.Name) + require.NoError(t, err, "could not load manifest") + + // Use a custom dockerfile template + m.Dockerfile = "Dockerfile.template" + customFrom := `FROM ubuntu:latest +# stuff +RUN random-command +# PORTER_INIT +COPY mybin /cnab/app/ +# No attempts are made to validate the contents +# Nor does it modify the contents in any way` + c.TestContext.AddTestFileContents([]byte(customFrom), "Dockerfile.template") + + mp := mixin.NewTestMixinProvider() + g := NewDockerfileGenerator(c.Config, m, tmpl, mp) + err = g.GenerateDockerFile(ctx) + require.NoError(t, err) + + wantDockerfilePath := ".cnab/Dockerfile" + gotDockerfile, err := c.FileSystem.ReadFile(wantDockerfilePath) + require.NoError(t, err) + + // Verify that we logged the dockerfile contents + tests.RequireOutputContains(t, c.TestContext.GetError(), string(gotDockerfile), "expected the dockerfile to be printed to the logs") + assert.Equal(t, customFrom, string(gotDockerfile)) +} + +func TestPorter_GenerateDockerfile_WithExperimentalFlagFullControlDockerfile_RequiresDockerfileSpecified(t *testing.T) { + t.Parallel() + + ctx := context.Background() + c := config.NewTestConfig(t) + + // Enable the experimental feature + c.SetExperimentalFlags(experimental.FlagFullControlDockerfile) + + defer c.Close() + + // Start a span so we can capture the output + ctx, log := c.StartRootSpan(ctx, t.Name()) + defer log.Close() + + tmpl := templates.NewTemplates(c.Config) + configTpl, err := tmpl.GetManifest() + require.Nil(t, err) + c.TestContext.AddTestFileContents(configTpl, config.Name) + + m, err := manifest.LoadManifestFrom(ctx, c.Config, config.Name) + require.NoError(t, err, "could not load manifest") + + mp := mixin.NewTestMixinProvider() + g := NewDockerfileGenerator(c.Config, m, tmpl, mp) + err = g.GenerateDockerFile(ctx) + + require.EqualError(t, err, "error reading the Dockerfile: no Dockerfile specified in the manifest") +} + +func TestPorter_GenerateDockerfile_WithExperimentalFlagFullControlDockerfile_DockerfileMustExist(t *testing.T) { + t.Parallel() + + ctx := context.Background() + c := config.NewTestConfig(t) + + // Enable the experimental feature + c.SetExperimentalFlags(experimental.FlagFullControlDockerfile) + + defer c.Close() + + // Start a span so we can capture the output + ctx, log := c.StartRootSpan(ctx, t.Name()) + defer log.Close() + + tmpl := templates.NewTemplates(c.Config) + configTpl, err := tmpl.GetManifest() + require.Nil(t, err) + c.TestContext.AddTestFileContents(configTpl, config.Name) + + m, err := manifest.LoadManifestFrom(ctx, c.Config, config.Name) + require.NoError(t, err, "could not load manifest") + + m.Dockerfile = "this-file-does-not-exist.dockerfile" + + mp := mixin.NewTestMixinProvider() + g := NewDockerfileGenerator(c.Config, m, tmpl, mp) + err = g.GenerateDockerFile(ctx) + + require.EqualError(t, err, "error reading the Dockerfile: the Dockerfile specified in the manifest doesn't exist: \"this-file-does-not-exist.dockerfile\"") +} diff --git a/pkg/experimental/experimental.go b/pkg/experimental/experimental.go index eef20640a..ff0809306 100644 --- a/pkg/experimental/experimental.go +++ b/pkg/experimental/experimental.go @@ -6,6 +6,9 @@ const ( // DependenciesV2 is the name of the experimental feature flag for PEP003 - Advanced Dependencies. DependenciesV2 = "dependencies-v2" + + // FullControlDockerfile is the name of the experimental feature flag giving authors full control of the invocation image Dockerfile + FullControlDockerfile = "full-control-dockerfile" ) // FeatureFlags is an enum of possible feature flags @@ -17,6 +20,9 @@ const ( // FlagDependenciesV2 gates the changes from PEP003 - Advanced Dependencies. FlagDependenciesV2 + + // FlagFullControlDockerfile gates the changes required for giving authors full control of the invocation image Dockerfile + FlagFullControlDockerfile ) // ParseFlags converts a list of feature flag names into a bit map for faster lookups. @@ -28,6 +34,8 @@ func ParseFlags(flags []string) FeatureFlags { experimental = experimental | FlagNoopFeature case DependenciesV2: experimental = experimental | FlagDependenciesV2 + case FullControlDockerfile: + experimental = experimental | FlagFullControlDockerfile } } return experimental From 1e718dd066a002c0ec293c9a4d28a0d827f40e82 Mon Sep 17 00:00:00 2001 From: Jens Arnfast Date: Tue, 14 May 2024 20:04:48 +0200 Subject: [PATCH 2/2] Adjusted documentation of full-control-dockerfile experimental flag Signed-off-by: Jens Arnfast --- docs/content/docs/configuration/configuration.md | 8 ++++---- .../concepts-and-components/intro-configuration.md | 9 ++++----- pkg/build/dockerfile-generator.go | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/content/docs/configuration/configuration.md b/docs/content/docs/configuration/configuration.md index b7d3dfa6d..e2c295fe7 100644 --- a/docs/content/docs/configuration/configuration.md +++ b/docs/content/docs/configuration/configuration.md @@ -282,11 +282,11 @@ When it is completed, it is used to activate the features from [PEP003 - Advance ### Full control Dockerfile The `full-control-dockerfile` experimental flag disables all Dockerfile generation when building bundles. -When enabled Porter will use the file referenced by `dockerfile` when building the invocation image without modifying it in any way. -This includes injection of any `# PORTER_x` templates, user configuration and `CMD` statements. -It is up to the bundle author to ensure that the file referenced by `dockerfile` contains the necessary tools for any mixins to function and a layout that can be executed as a Porter bundle. +When enabled Porter will use the file referenced by `dockerfile` in the Porter manifest when building the invocation image *without modifying* it in any way. +Ie. Porter will not process `# PORTER_x` placeholders, nor inject any user configuration and `CMD` statements. +It is up to the bundle author to ensure that the contents of the Dockerfile contains the necessary tools for any mixins to function and a layout that can be executed as a Porter bundle. -Note that `autobuild` does not detect changes to the contents of the file referenced by `dockerfile`. +*Note:* when using the `dockerfile` property in the Porter manifest the `autobuild` functionality does not re-build the bundle on changes to the contents of the referenced file. ## Common Configuration Settings diff --git a/docs/content/docs/introduction/concepts-and-components/intro-configuration.md b/docs/content/docs/introduction/concepts-and-components/intro-configuration.md index f7236c33d..916b94c21 100644 --- a/docs/content/docs/introduction/concepts-and-components/intro-configuration.md +++ b/docs/content/docs/introduction/concepts-and-components/intro-configuration.md @@ -279,12 +279,11 @@ When it is completed, it is used to activate the features from [PEP003 - Advance ### Full control Dockerfile The `full-control-dockerfile` experimental flag disables all Dockerfile generation when building bundles. -When enabled Porter will use the file referenced by `dockerfile` when building the invocation image without modifying it in any way. -This includes injection of any `# PORTER_x` templates, user configuration and `CMD` statements. -It is up to the bundle author to ensure that the file referenced by `dockerfile` contains the necessary tools for any mixins to function and a layout that can be executed as a Porter bundle. - -Note that `autobuild` does not detect changes to the contents of the file referenced by `dockerfile`. +When enabled Porter will use the file referenced by `dockerfile` in the Porter manifest when building the invocation image *without modifying* it in any way. +Ie. Porter will not process `# PORTER_x` placeholders, nor inject any user configuration and `CMD` statements. +It is up to the bundle author to ensure that the contents of the Dockerfile contains the necessary tools for any mixins to function and a layout that can be executed as a Porter bundle. +*Note:* when using the `dockerfile` property in the Porter manifest the `autobuild` functionality does not re-build the bundle on changes to the contents of the referenced file. ## Common Configuration Settings Some configuration settings are applicable to many of Porter's commands and to save time you may want to set these values in the configuration file or with environment variables. diff --git a/pkg/build/dockerfile-generator.go b/pkg/build/dockerfile-generator.go index 1922f7d57..45b2e2cd2 100644 --- a/pkg/build/dockerfile-generator.go +++ b/pkg/build/dockerfile-generator.go @@ -50,7 +50,7 @@ func (g *DockerfileGenerator) GenerateDockerFile(ctx context.Context) error { var lines []string var err error if g.Config.IsFeatureEnabled(experimental.FlagFullControlDockerfile) { - span.Warnf("Experimental feature \"%s\" enabled: Dockerfile will be used without changes by Porter", + span.Warnf("WARNING: Experimental feature \"%s\" enabled: Dockerfile will be used without changes by Porter", experimental.FullControlDockerfile) lines, err = g.readRawDockerfile(ctx) if err != nil {