From 1e899b5401084862574cd091715cff312416d676 Mon Sep 17 00:00:00 2001 From: Oran Moshai <12291998+oranmoshai@users.noreply.github.com> Date: Sun, 10 Apr 2022 12:25:33 +0300 Subject: [PATCH] Oran/image scan (#69) * Add: support for image scanning This will assume repositry name and id is full registry url for example: 1111111.dkr.ecr.us-east-1.amazonaws.com/alpine And also inject tag as branch name Currently support trivy standalone with no server caching Co-authored-by: oranmoshai --- cmd/aqua/main.go | 7 +- go.mod | 1 + pkg/buildClient/client.go | 6 +- pkg/buildClient/policies.go | 2 +- pkg/buildClient/repository.go | 31 ++++--- pkg/buildClient/upload.go | 2 +- pkg/metadata/collector.go | 130 +++++++++++++++++++++++++----- pkg/metadata/collector_test.go | 106 +++++++++++++++++++++++- pkg/processor/result_processor.go | 9 ++- pkg/scanner/local_scanner.go | 1 + pkg/scanner/scanner.go | 45 ++++++++++- test/metadata_test.go | 2 +- 12 files changed, 301 insertions(+), 41 deletions(-) diff --git a/cmd/aqua/main.go b/cmd/aqua/main.go index 905008ed..5081d531 100644 --- a/cmd/aqua/main.go +++ b/cmd/aqua/main.go @@ -82,15 +82,18 @@ func main() { }, ) + imageCmd := commands.NewImageCommand() + imageCmd.Action = runScan + app.Action = runScan app.Flags = fsCmd.Flags app.Commands = []*cli.Command{ fsCmd, configCmd, + imageCmd, commands.NewPluginCommand(), commands.NewClientCommand(), - commands.NewImageCommand(), commands.NewRepositoryCommand(), commands.NewRootfsCommand(), commands.NewServerCommand(), @@ -125,7 +128,7 @@ func runScan(c *cli.Context) error { } log.Logger.Debugf("Using scanPath %s", scanPath) - client, err := buildClient.Get(scanPath) + client, err := buildClient.Get(scanPath, c) if err != nil { return err } diff --git a/go.mod b/go.mod index dde51107..b62370b7 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/twitchtv/twirp v8.1.1+incompatible github.com/urfave/cli/v2 v2.3.0 go.uber.org/zap v1.20.0 + golang.org/x/text v0.3.7 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 google.golang.org/protobuf v1.27.1 ) diff --git a/pkg/buildClient/client.go b/pkg/buildClient/client.go index 0cc06125..137be892 100644 --- a/pkg/buildClient/client.go +++ b/pkg/buildClient/client.go @@ -6,6 +6,8 @@ import ( "net/http" "os" + "github.com/urfave/cli/v2" + "github.com/aquasecurity/trivy-plugin-aqua/pkg/log" "github.com/aquasecurity/trivy-plugin-aqua/pkg/proto/buildsecurity" "github.com/pkg/errors" @@ -20,6 +22,7 @@ type Client interface { type TwirpClient struct { client buildsecurity.BuildSecurity + c *cli.Context scanPath string jwtToken string aquaUrl string @@ -28,7 +31,7 @@ type TwirpClient struct { var buildClient Client -func Get(scanPath string) (Client, error) { +func Get(scanPath string, c *cli.Context) (Client, error) { if buildClient != nil { log.Logger.Debugf("Valid client found, re-using...") return buildClient, nil @@ -67,6 +70,7 @@ func Get(scanPath string) (Client, error) { scanPath: scanPath, jwtToken: jwtToken, aquaUrl: aquaURL, + c: c, } return buildClient, nil diff --git a/pkg/buildClient/policies.go b/pkg/buildClient/policies.go index 8d8c5e1d..56206c02 100644 --- a/pkg/buildClient/policies.go +++ b/pkg/buildClient/policies.go @@ -18,7 +18,7 @@ func (bc *TwirpClient) GetPoliciesForRepository() ([]*buildsecurity.Policy, erro return nil, err } - _, branch, err := metadata.GetRepositoryDetails(bc.scanPath) + _, branch, err := metadata.GetRepositoryDetails(bc.scanPath, bc.c.Command.Name) if err != nil { return nil, err } diff --git a/pkg/buildClient/repository.go b/pkg/buildClient/repository.go index 989fde49..4e66331f 100644 --- a/pkg/buildClient/repository.go +++ b/pkg/buildClient/repository.go @@ -1,6 +1,8 @@ package buildClient import ( + "fmt" + "github.com/aquasecurity/trivy-plugin-aqua/pkg/log" "github.com/aquasecurity/trivy-plugin-aqua/pkg/metadata" "github.com/aquasecurity/trivy-plugin-aqua/pkg/proto/buildsecurity" @@ -57,20 +59,31 @@ func (bc *TwirpClient) GetOrCreateRepository() (string, error) { return repoId, nil } -func (bc *TwirpClient) getScmID() (string, error) { - scmID, err := metadata.GetScmID(bc.scanPath) - if err != nil { - return "", err +func (bc *TwirpClient) getScmID() (scmID string, err error) { + switch bc.c.Command.Name { + case "image": + prefix, repo, _ := metadata.GetImageDetails(bc.scanPath) + scmID = metadata.GetRepositoryUrl(prefix, repo) + default: + scmID, err = metadata.GetScmID(bc.scanPath) + if err != nil { + return "", fmt.Errorf("failed get scm id: %w", err) + } } return scmID, nil } -func (bc *TwirpClient) getRepoName() (string, error) { - repoName, _, err := metadata.GetRepositoryDetails(bc.scanPath) - if err != nil { - return "", err +func (bc *TwirpClient) getRepoName() (repoName string, err error) { + switch bc.c.Command.Name { + case "image": + prefix, repo, _ := metadata.GetImageDetails(bc.scanPath) + repoName = metadata.GetRepositoryUrl(prefix, repo) + default: + repoName, _, err = metadata.GetRepositoryDetails(bc.scanPath, "") + if err != nil { + return "", err + } } - return repoName, nil } diff --git a/pkg/buildClient/upload.go b/pkg/buildClient/upload.go index c54c5102..bd8bc632 100644 --- a/pkg/buildClient/upload.go +++ b/pkg/buildClient/upload.go @@ -17,7 +17,7 @@ func (bc *TwirpClient) Upload(results []*buildsecurity.Result, tags map[string]s } gitUser := metadata.GetGitUser(bc.scanPath) - _, branch, err := metadata.GetRepositoryDetails(bc.scanPath) + _, branch, err := metadata.GetRepositoryDetails(bc.scanPath, bc.c.Command.Name) if err != nil { return err } diff --git a/pkg/metadata/collector.go b/pkg/metadata/collector.go index fa68ed99..af222975 100644 --- a/pkg/metadata/collector.go +++ b/pkg/metadata/collector.go @@ -2,6 +2,7 @@ package metadata import ( "fmt" + "net" "io/ioutil" "os" @@ -12,6 +13,74 @@ import ( "github.com/aquasecurity/trivy-plugin-aqua/pkg/log" ) +var ( + // SHARegexp is used to split an image digest value to a registry prefix, + // repository name, and the SHA256 hash. + SHARegexp = regexp.MustCompile(`^(?:([^/]+)/)([^@]+)(@sha256:[0-9a-f]+)$`) + + // SplitImageNameRegexp is used to split a fully qualified image name to a + // registry prefix, repository name and image tag. + SplitImageNameRegexp = regexp.MustCompile(`^(?:([^/]+)/)?([^:]+)(?::(.*))?$`) + // PortRegexp is used to check whether a string ends with a port suffix + // (e.g. :8080) + PortRegexp = regexp.MustCompile(`:\d+$`) +) + +func GetRepositoryUrl(prefix, repo string) string { + if prefix != "" { + return fmt.Sprintf("%s/%s", prefix, repo) + } + return repo +} + +// GetImageDetails gets the full name of an image (e.g. "repo/test:master", +// or even "docker.io/repo/test:master") and splits it into the registry +// prefix, repository name and the image tag (e.g. "docker.io", "repo/test" +// and "master" in the previous example). +func GetImageDetails(imageName string) (prefix, repo, tag string) { + if imageName == "" { + return prefix, repo, tag + } + + shaMatches := SHARegexp.FindStringSubmatch(imageName) + if len(shaMatches) == 4 { + prefix = shaMatches[1] + repo = shaMatches[2] + tag = shaMatches[3] + } else { + matches := SplitImageNameRegexp.FindStringSubmatch(imageName) + if len(matches) < 3 { + return prefix, repo, tag + } + + prefix = matches[1] + repo = matches[2] + tag = matches[3] + } + + // we may have extracted a prefix, but it may actually be part of the + // repository name, because repository names can contain multiple slashes. + // to verify, we will check that the prefix we extract is a valid IP address + // or DNS name. We will also assume everything with a port suffix (e.g. + // :8080) is a registry prefix + if prefix != "" && !PortRegexp.MatchString(prefix) { + if net.ParseIP(prefix) == nil { + dns, _ := net.LookupIP(prefix) + if len(dns) == 0 { + // prefix is probably a part of the repository name + if repo == "" { + repo = prefix + } else { + repo = fmt.Sprintf("%s/%s", prefix, repo) + } + prefix = "" + } + } + } + + return prefix, repo, tag +} + // GetScmID extracts the git path from the config file func GetScmID(scanPath string) (string, error) { gitConfigFile := filepath.Join(scanPath, ".git", "config") @@ -53,28 +122,8 @@ func GetBuildSystem() string { return "other" } -// GetRepositoryDetails gets the repository name and branch -// multiple env vars will be checked first before falling back to the folder name -func GetRepositoryDetails(scanPath string) (repoName, branch string, err error) { - - for _, repoEnv := range possibleRepoEnvVars { - if v, ok := os.LookupEnv(repoEnv); ok { - repoName = v - break - } - } - - for _, branchEnv := range possibleBranchEnvVars { - if v, ok := os.LookupEnv(branchEnv); ok { - branch = v - break - } - } - - if repoName != "" && branch != "" { - return repoName, branch, nil - } - +// Get repository details based on FS scan +func getFsRepositoryDetails(scanPath string) (repoName, branch string, err error) { workingDir := scanPath abs, err := filepath.Abs(workingDir) if err != nil { @@ -107,6 +156,43 @@ func GetRepositoryDetails(scanPath string) (repoName, branch string, err error) return inferredRepoName, "", nil } +// GetRepositoryDetails gets the repository name and branch +// multiple env vars will be checked first before falling back to the folder name +func GetRepositoryDetails(scanPath string, cmd string) (repoName, branch string, err error) { + + for _, repoEnv := range possibleRepoEnvVars { + if v, ok := os.LookupEnv(repoEnv); ok { + repoName = v + break + } + } + + for _, branchEnv := range possibleBranchEnvVars { + if v, ok := os.LookupEnv(branchEnv); ok { + branch = v + break + } + } + + if repoName != "" && branch != "" { + return repoName, branch, nil + } + + switch cmd { + case "image": + prefix, repo, tag := GetImageDetails(scanPath) + branch = tag + repoName = GetRepositoryUrl(prefix, repo) + default: + repoName, branch, err = getFsRepositoryDetails(scanPath) + if err != nil { + return repoName, branch, fmt.Errorf("failed get FS repository details: %w", err) + } + } + + return repoName, branch, nil +} + // GetCommitID gets the current CommitID of the repository func GetCommitID(scanPath string) (commitId string) { diff --git a/pkg/metadata/collector_test.go b/pkg/metadata/collector_test.go index f51c2153..90676561 100644 --- a/pkg/metadata/collector_test.go +++ b/pkg/metadata/collector_test.go @@ -1,6 +1,8 @@ package metadata -import "testing" +import ( + "testing" +) func Test_convertScmId(t *testing.T) { type args struct { @@ -56,3 +58,105 @@ func Test_convertScmId(t *testing.T) { }) } } + +func TestGetRepositoryUrl(t *testing.T) { + type args struct { + prefix string + repo string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "happy path", + args: args{prefix: "prefix", repo: "repo"}, + want: "prefix/repo", + }, + { + name: "happy path - only repo", + args: args{repo: "repo"}, + want: "repo", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetRepositoryUrl(tt.args.prefix, tt.args.repo); got != tt.want { + t.Errorf("GetRepositoryUrl() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetImageDetails(t *testing.T) { + type args struct { + imageName string + } + tests := []struct { + name string + args args + wantPrefix string + wantRepo string + wantTag string + }{ + { + name: "happy path - full docker url", + args: args{imageName: "docker.io/repo/test:master"}, + wantPrefix: "docker.io", + wantRepo: "repo/test", + wantTag: "master", + }, + { + name: "happy path - docker url", + args: args{imageName: "repo/test:master"}, + wantPrefix: "", + wantRepo: "repo/test", + wantTag: "master", + }, + { + name: "happy path - docker full url two /", + args: args{imageName: "docker.io/library/centos:latest"}, + wantPrefix: "docker.io", + wantRepo: "library/centos", + wantTag: "latest", + }, + { + name: "happy path - docker url two", + args: args{imageName: "library/centos:latest"}, + wantPrefix: "", + wantRepo: "library/centos", + wantTag: "latest", + }, + { + name: "happy path - aws ecr", + args: args{imageName: "1111111.dkr.ecr.us-east-1.amazonaws.com/alpine:3.9.4"}, + wantPrefix: "1111111.dkr.ecr.us-east-1.amazonaws.com", + wantRepo: "alpine", + wantTag: "3.9.4", + }, + { + name: "happy path - docker hash", + args: args{ + imageName: "docker.io/repo/test@sha256:715760eedeabb0ca7b5758d4536e78c4c06cad699caa912bf1ef0f483b103efc", + }, + wantPrefix: "docker.io", + wantRepo: "repo/test", + wantTag: "@sha256:715760eedeabb0ca7b5758d4536e78c4c06cad699caa912bf1ef0f483b103efc", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPrefix, gotRepo, gotTag := GetImageDetails(tt.args.imageName) + if gotPrefix != tt.wantPrefix { + t.Errorf("GetImageDetails() gotPrefix = %v, want %v", gotPrefix, tt.wantPrefix) + } + if gotRepo != tt.wantRepo { + t.Errorf("GetImageDetails() gotRepo = %v, want %v", gotRepo, tt.wantRepo) + } + if gotTag != tt.wantTag { + t.Errorf("GetImageDetails() gotTag = %v, want %v", gotTag, tt.wantTag) + } + }) + } +} diff --git a/pkg/processor/result_processor.go b/pkg/processor/result_processor.go index 19e9cda9..d7a37fd5 100644 --- a/pkg/processor/result_processor.go +++ b/pkg/processor/result_processor.go @@ -4,6 +4,9 @@ import ( "fmt" "strings" + "golang.org/x/text/cases" + "golang.org/x/text/language" + "github.com/aquasecurity/trivy-plugin-aqua/pkg/log" "github.com/aquasecurity/trivy-plugin-aqua/pkg/proto/buildsecurity" "github.com/aquasecurity/trivy-plugin-aqua/pkg/scanner" @@ -26,6 +29,9 @@ func ProcessResults(reports report.Results, case report.ClassConfig: reportResults := addMisconfigurationResults(rep, policies, checkSupIDMap) results = append(results, reportResults...) + case report.ClassOSPkg: + reportResults := addVulnerabilitiesResults(rep) + results = append(results, reportResults...) } } @@ -109,7 +115,8 @@ func addMisconfigurationResults(rep report.Result, for _, miscon := range rep.Misconfigurations { var r buildsecurity.Result - resource := fmt.Sprintf("%s Resource", strings.Title(rep.Type)) + resource := fmt.Sprintf("%s Resource", cases.Title(language.English).String(rep.Type)) + if miscon.IacMetadata.Resource != "" { resource = miscon.IacMetadata.Resource } diff --git a/pkg/scanner/local_scanner.go b/pkg/scanner/local_scanner.go index efc49dcd..858684f5 100644 --- a/pkg/scanner/local_scanner.go +++ b/pkg/scanner/local_scanner.go @@ -37,6 +37,7 @@ func (s aquaScanner) Scan(target, imageID string, layerIDs []string, options typ if err != nil { return nil, osFound, err } + for i := range results { s.resultClient.FillVulnerabilityInfo(results[i].Vulnerabilities, results[i].Type) } diff --git a/pkg/scanner/scanner.go b/pkg/scanner/scanner.go index 2fa63463..7ea8f921 100644 --- a/pkg/scanner/scanner.go +++ b/pkg/scanner/scanner.go @@ -9,8 +9,13 @@ import ( "os" "strings" + "github.com/aquasecurity/fanal/image" + "github.com/aquasecurity/fanal/types" + + analyzerConfig "github.com/aquasecurity/fanal/analyzer/config" fanalconfig "github.com/aquasecurity/fanal/analyzer/config" fanalartifact "github.com/aquasecurity/fanal/artifact" + image2 "github.com/aquasecurity/fanal/artifact/image" "github.com/aquasecurity/fanal/artifact/local" "github.com/aquasecurity/fanal/cache" "github.com/aquasecurity/trivy-plugin-aqua/pkg/proto/buildsecurity" @@ -28,8 +33,13 @@ const ( ) func Scan(c *cli.Context, path string) (report.Results, error) { - - initializeScanner := initializeFilesystemScanner(path, policyDir, dataDir) + var initializeScanner artifact.InitializeScanner + switch c.Command.Name { + case "image": + initializeScanner = initializeDockerScanner(path) + default: + initializeScanner = initializeFilesystemScanner(path, policyDir, dataDir) + } opt, err := createScanOptions(c) if err != nil { @@ -68,6 +78,37 @@ func Scan(c *cli.Context, path string) (report.Results, error) { } +func initializeDockerScanner(path string) artifact.InitializeScanner { + return func( + ctx context.Context, + s string, + artifactCache cache.ArtifactCache, + localArtifactCache cache.LocalArtifactCache, + b bool, + option fanalartifact.Option, + option2 fanalconfig.ScannerOption) ( + scanner.Scanner, func(), error) { + localScanner := newAquaScanner(localArtifactCache) + typesImage, cleanup, err := image.NewDockerImage(ctx, path, types.DockerOption{}) + if err != nil { + return scanner.Scanner{}, nil, err + } + artifactArtifact, err := image2.NewArtifact( + typesImage, + artifactCache, + fanalartifact.Option{}, + analyzerConfig.ScannerOption{}) + if err != nil { + cleanup() + return scanner.Scanner{}, nil, err + } + scannerScanner := scanner.NewScanner(localScanner, artifactArtifact) + return scannerScanner, func() { + cleanup() + }, nil + } +} + func initializeFilesystemScanner(dir, _, _ string) artifact.InitializeScanner { return func(_ context.Context, _ string, artifactCache cache.ArtifactCache, diff --git a/test/metadata_test.go b/test/metadata_test.go index c15629b9..0c4e1657 100644 --- a/test/metadata_test.go +++ b/test/metadata_test.go @@ -37,7 +37,7 @@ func Test_get_repo_name_from_env(t *testing.T) { err = os.Setenv(possibleBranchEnvVars[i], fmt.Sprintf("BRANCH_%d", i)) require.NoError(t, err) - repoName, branch, err := metadata.GetRepositoryDetails("") + repoName, branch, err := metadata.GetRepositoryDetails("", "") require.NoError(t, err) assert.Equal(t, fmt.Sprintf("REPOSITORY_%d", i), repoName)