feat: Add show-builtin-errors flag for the verify command (#901)

This is useful to raise config parsing errors when using the parse_config
builtins. Previously, the unit test would fail but it was unclear to the
user whether that was due to an error in the policy logic or a typo in
the config.

Signed-off-by: James Alseth <james@jalseth.me>

docs/index.md

@@ -127,6 +127,11 @@ uses when testing configurations, only exposed as a Rego function. The example
 below shows how to use this to parse an AWS Terraform configuration and use it
 in a unit test.
 
+> **TIP:** It is recommended to use the `--show-builtin-errors` flag when
+> using the `parse_config`, `parse_config_file`, and `parse_combined_config_files`
+> functions. This way errors encountered during parsing will be raised. This
+> flag will be enabled by default in a future release.
+
 **deny.rego**
 
 ```rego

internal/commands/verify.go

@@ -80,6 +80,7 @@ func NewVerifyCommand(ctx context.Context) *cobra.Command {
 				"capabilities",
 				"strict",
 				"proto-file-dirs",
+				"show-builtin-errors",
 			}
 			for _, name := range flagNames {
 				if err := viper.BindPFlag(name, cmd.Flags().Lookup(name)); err != nil {
@@ -137,6 +138,7 @@ func NewVerifyCommand(ctx context.Context) *cobra.Command {
 	cmd.Flags().Bool("trace", false, "Enable more verbose trace output for Rego queries")
 	cmd.Flags().Bool("strict", false, "Enable strict mode for Rego policies")
 	cmd.Flags().String("report", "", "Shows output for Rego queries as a report with summary. Available options are {full|notes|fails}.")
+	cmd.Flags().Bool("show-builtin-errors", false, "Collect and return all encountered built-in errors")
 
 	cmd.Flags().StringP("output", "o", output.OutputStandard, fmt.Sprintf("Output format for conftest results - valid options are: %s", output.Outputs()))
 	cmd.Flags().Bool("junit-hide-message", false, "Do not include the violation message in the JUnit test name")

runner/verify.go

@@ -15,15 +15,16 @@ import (
 // VerifyRunner is the runner for the Verify command, executing
 // Rego policy unit-tests.
 type VerifyRunner struct {
-	Capabilities string
-	Policy       []string
-	Data         []string
-	Output       string
-	NoColor      bool `mapstructure:"no-color"`
-	Trace        bool
-	Strict       bool
-	Report       string
-	Quiet        bool
+	Capabilities      string
+	Policy            []string
+	Data              []string
+	Output            string
+	NoColor           bool `mapstructure:"no-color"`
+	Trace             bool
+	Strict            bool
+	Report            string
+	Quiet             bool
+	ShowBuiltinErrors bool `mapstructure:"show-builtin-errors"`
 }
 
 const (
@@ -51,7 +52,8 @@ func (r *VerifyRunner) Run(ctx context.Context) ([]output.CheckResult, []*tester
 		SetStore(engine.Store()).
 		SetModules(engine.Modules()).
 		EnableTracing(enableTracing).
-		SetRuntime(engine.Runtime())
+		SetRuntime(engine.Runtime()).
+		RaiseBuiltinErrors(r.ShowBuiltinErrors)
 	ch, err := runner.RunTests(ctx, nil)
 	if err != nil {
 		return nil, nil, fmt.Errorf("running tests: %w", err)

tests/builtin-errors/policy/main.rego

@@ -0,0 +1,6 @@
+package main
+
+deny[{"msg": msg}] {
+    input.test_field == 123
+    msg := "some error"
+}

tests/builtin-errors/policy/main_test.rego

@@ -0,0 +1,5 @@
+package main
+
+test_deny_valid {
+    not deny with input as parse_config_file("file_does_not_exist.yaml")
+}

tests/builtin-errors/test.bats

@@ -0,0 +1,17 @@
+#!/usr/bin/env bats
+
+@test "Parsing error without show-builtin-errors flag returns test failed" {
+  run $CONFTEST verify --show-builtin-errors=false
+
+  [ "$status" -eq 1 ]
+  echo $output
+  [[ "$output" =~ "1 test, 0 passed, 0 warnings, 1 failure, 0 exceptions, 0 skipped" ]]
+}
+
+@test "Parsing error with show-builtin-errors flag returns builtin error" {
+  run $CONFTEST verify --show-builtin-errors=true
+
+  [ "$status" -eq 1 ]
+  echo $output
+  [[ "$output" =~ "file_does_not_exist.yaml: no such file or directory" ]]
+}