From bba0267871bb25a17e8dd2fac318fb3fa846090e Mon Sep 17 00:00:00 2001 From: Tomoya Amachi Date: Fri, 10 Sep 2021 13:22:59 +0900 Subject: [PATCH] check suspitious file extensions and add suspitious filenames (#150) * check files with file extension and add accept-key option * add accept-file-extension option * update README --- README.md | 25 ++++++- cmd/dockle/main.go | 31 +++++--- pkg/assessor/assessor.go | 8 ++ pkg/assessor/cache/cache.go | 4 + pkg/assessor/contentTrust/contentTrust.go | 6 +- pkg/assessor/credential/credential.go | 91 +++++++++++++++++++---- pkg/assessor/group/group.go | 4 + pkg/assessor/hosts/hosts.go | 6 +- pkg/assessor/manifest/manifest.go | 4 + pkg/assessor/passwd/passwd.go | 4 + pkg/assessor/privilege/suid.go | 4 + pkg/assessor/user/user.go | 4 + pkg/run.go | 3 +- pkg/scanner/scan.go | 79 +++++++++++++------- pkg/scanner/scan_test.go | 1 + 15 files changed, 222 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 1433325..71c2c9a 100644 --- a/README.md +++ b/README.md @@ -627,7 +627,14 @@ The `--ignore, -i` option can ignore specified checkpoints. $ dockle -i CIS-DI-0001 -i DKL-DI-0006 [IMAGE_NAME] ``` -Or, use `.dockleignore` file. +Or, use `DOCKLE_IGNORS`: + +``` +export DOCKLE_IGNORES=CIS-DI-0001,DKL-DI-0006 +dockle [IMAGE_NAME] +``` + +Or, use `.dockleignore` file: ```bash $ cat .dockleignore @@ -637,6 +644,22 @@ CIS-DI-0001 DKL-DI-0006 ``` +### Accept suspitious `environment variables` / `files` / `file extensions` + +```bash +# --accept-key value, --ak value You can add acceptable keywords. +dockle -ak GPG_KEY -ak KEYCLOAK_VERSION [IMAGE_NAME] +or DOCKLE_ACCEPT_KEYS=GPG_KEY,KEYCLOAK_VERSION dockle [IMAGE_NAME] + +# --accept-file value, --af value You can add acceptable file names. +dockle -af id_rsa -af id_dsa [IMAGE_NAME] +or DOCKLE_ACCEPT_FILES=id_rsa,id_dsa dockle [IMAGE_NAME] + +# --accept-file-extension value, --ae value You can add acceptable file extensions. +dockle -ae pem -ae log [IMAGE_NAME] +or DOCKLE_ACCEPT_FILE_EXTENSIONS=pem,log dockle [IMAGE_NAME] +``` + ## Continuous Integration (CI) You can scan your built image with `Dockle` in Travis CI/CircleCI. diff --git a/cmd/dockle/main.go b/cmd/dockle/main.go index cbd95bc..b16bc12 100644 --- a/cmd/dockle/main.go +++ b/cmd/dockle/main.go @@ -43,13 +43,24 @@ OPTIONS: Usage: "input file path instead of image name", }, cli.StringSliceFlag{ - Name: "ignore, i", - Usage: "checkpoints to ignore. You can use .dockleignore too.", + Name: "ignore, i", + EnvVar: "DOCKLE_IGNORES", + Usage: "checkpoints to ignore. You can use .dockleignore too.", }, cli.StringSliceFlag{ - Name: "accept-key, a", - EnvVar: "ACCEPT_KEY", - Usage: "For CIS-DI-0010. You can add acceptable keywords. e.g) -a GPG_KEY -a KEYCLOAK", + Name: "accept-key, ak", + EnvVar: "DOCKLE_ACCEPT_KEYS", + Usage: "For CIS-DI-0010. You can add acceptable keywords. e.g) -ak GPG_KEY -ak KEYCLOAK", + }, + cli.StringSliceFlag{ + Name: "accept-file, af", + EnvVar: "DOCKLE_ACCEPT_FILES", + Usage: "For CIS-DI-0010. You can add acceptable file names. e.g) -af id_rsa -af config.json", + }, + cli.StringSliceFlag{ + Name: "accept-file-extension, ae", + EnvVar: "DOCKLE_ACCEPT_FILE_EXTENSIONS", + Usage: "For CIS-DI-0010. You can add acceptable file extensions. e.g) -ae pem -ae log", }, cli.StringFlag{ Name: "format, f", @@ -79,17 +90,17 @@ OPTIONS: Usage: "suppress log output", }, cli.BoolFlag{ - Name: "no-color", + Name: "no-color", EnvVar: "NO_COLOR", - Usage: "suppress log output", + Usage: "suppress log output", }, // Registry flag cli.DurationFlag{ - Name: "timeout, t", - Value: time.Second * 90, + Name: "timeout, t", + Value: time.Second * 90, EnvVar: "DOCKLE_TIMEOUT", - Usage: "docker timeout. e.g) 5s, 5m...", + Usage: "docker timeout. e.g) 5s, 5m...", }, cli.StringFlag{ Name: "authurl", diff --git a/pkg/assessor/assessor.go b/pkg/assessor/assessor.go index 7b6fb2e..d0933a2 100644 --- a/pkg/assessor/assessor.go +++ b/pkg/assessor/assessor.go @@ -26,6 +26,7 @@ var assessors []Assessor type Assessor interface { Assess(deckodertypes.FileMap) ([]*types.Assessment, error) RequiredFiles() []string + RequiredExtensions() []string RequiredPermissions() []os.FileMode } @@ -63,6 +64,13 @@ func LoadRequiredFiles() (filenames []string) { return filenames } +func LoadRequiredExtensions() (extensions []string) { + for _, assessor := range assessors { + extensions = append(extensions, assessor.RequiredExtensions()...) + } + return extensions +} + func LoadRequiredPermissions() (permissions []os.FileMode) { for _, assessor := range assessors { permissions = append(permissions, assessor.RequiredPermissions()...) diff --git a/pkg/assessor/cache/cache.go b/pkg/assessor/cache/cache.go index 6c2e39d..7e5e0f2 100644 --- a/pkg/assessor/cache/cache.go +++ b/pkg/assessor/cache/cache.go @@ -81,6 +81,10 @@ func (a CacheAssessor) RequiredFiles() []string { return append(reqFiles, reqDirs...) } +func (a CacheAssessor) RequiredExtensions() []string { + return []string{} +} + func (a CacheAssessor) RequiredPermissions() []os.FileMode { return []os.FileMode{} } diff --git a/pkg/assessor/contentTrust/contentTrust.go b/pkg/assessor/contentTrust/contentTrust.go index 5763bb0..61f60c5 100644 --- a/pkg/assessor/contentTrust/contentTrust.go +++ b/pkg/assessor/contentTrust/contentTrust.go @@ -13,7 +13,7 @@ var HostEnvironmentFileName = "ENVIRONMENT variable on HOST OS" type ContentTrustAssessor struct{} -func (a ContentTrustAssessor) Assess(fileMap deckodertypes.FileMap) ([]*types.Assessment, error) { +func (a ContentTrustAssessor) Assess(_ deckodertypes.FileMap) ([]*types.Assessment, error) { log.Logger.Debug("Scan start : DOCKER_CONTENT_TRUST") if os.Getenv("DOCKER_CONTENT_TRUST") != "1" { @@ -32,6 +32,10 @@ func (a ContentTrustAssessor) RequiredFiles() []string { return []string{} } +func (a ContentTrustAssessor) RequiredExtensions() []string { + return []string{} +} + func (a ContentTrustAssessor) RequiredPermissions() []os.FileMode { return []os.FileMode{} } diff --git a/pkg/assessor/credential/credential.go b/pkg/assessor/credential/credential.go index 22977cd..357045d 100644 --- a/pkg/assessor/credential/credential.go +++ b/pkg/assessor/credential/credential.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "unicode/utf8" deckodertypes "github.com/goodwithtech/deckoder/types" @@ -17,28 +18,92 @@ type CredentialAssessor struct{} func (a CredentialAssessor) Assess(fileMap deckodertypes.FileMap) ([]*types.Assessment, error) { log.Logger.Debug("Start scan : credential files") assesses := []*types.Assessment{} - reqFiles := a.RequiredFiles() + fmap := makeMaps(a.RequiredFiles()) + fexts := makeMaps(a.RequiredExtensions()) for filename := range fileMap { basename := filepath.Base(filename) // check exist target files - for _, reqFilename := range reqFiles { - if reqFilename == basename { - assesses = append( - assesses, - &types.Assessment{ - Code: types.AvoidCredential, - Filename: filename, - Desc: fmt.Sprintf("Suspicious filename found : %s ", filename), - }) - break - } + if _, ok := fmap[basename]; ok { + assesses = append( + assesses, + &types.Assessment{ + Code: types.AvoidCredential, + Filename: filename, + Desc: fmt.Sprintf("Suspicious filename found : %s (You can suppress it with \"-af %s\")", filename, basename), + }) + } else if _, ok := fexts[filepath.Ext(basename)]; ok { + assesses = append( + assesses, + &types.Assessment{ + Code: types.AvoidCredential, + Filename: filename, + Desc: fmt.Sprintf("Suspicious file extension found : %s (You can suppress it with \"-ae %s\")", filename, trimFirstRune(filepath.Ext(basename))), + }) } } return assesses, nil } +func trimFirstRune(s string) string { + _, i := utf8.DecodeRuneInString(s) + return s[i:] +} + +func makeMaps(keys []string) map[string]struct{} { + maps := make(map[string]struct{}) + for i := 0; i < len(keys); i++ { + maps[keys[i]] = struct{}{} + } + return maps +} + func (a CredentialAssessor) RequiredFiles() []string { - return []string{"credentials.json", "credential.json", "credentials", "credential"} + return []string{ + "credentials.json", + "credential.json", + "config.json", + "credentials", + "credential", + "password.txt", + "id_rsa", + "id_dsa", + "id_ecdsa", + "id_ed25519", + "secret_token.rb", + "carrierwave.rb", + "omniauth.rb", + "settings.py", + "database.yml", + "credentials.xml", + } +} + +func (a CredentialAssessor) RequiredExtensions() []string { + // reference: https://github.com/eth0izzle/shhgit/blob/master/config.yaml + return []string{ + ".key", + ".secret", + ".pem", + ".p12", + ".pkcs12", + ".pfx", + ".asc", + ".ovpn", + ".private_key", + ".cscfg", + ".rdp", + ".mdf", + ".sdf", + ".bek", + ".tpm", + ".fve", + ".jks", + ".psafe3", + ".agilekeychain", + ".keychain", + ".pcap", + ".gnucache", + } } func (a CredentialAssessor) RequiredPermissions() []os.FileMode { diff --git a/pkg/assessor/group/group.go b/pkg/assessor/group/group.go index a2cd490..ba85446 100644 --- a/pkg/assessor/group/group.go +++ b/pkg/assessor/group/group.go @@ -65,6 +65,10 @@ func (a GroupAssessor) RequiredFiles() []string { return []string{"etc/group"} } +func (a GroupAssessor) RequiredExtensions() []string { + return []string{} +} + func (a GroupAssessor) RequiredPermissions() []os.FileMode { return []os.FileMode{} } diff --git a/pkg/assessor/hosts/hosts.go b/pkg/assessor/hosts/hosts.go index 30c887a..1c5472f 100644 --- a/pkg/assessor/hosts/hosts.go +++ b/pkg/assessor/hosts/hosts.go @@ -11,7 +11,7 @@ import ( type HostsAssessor struct{} -func (a HostsAssessor) Assess(fileMap deckodertypes.FileMap) ([]*types.Assessment, error) { +func (a HostsAssessor) Assess(_ deckodertypes.FileMap) ([]*types.Assessment, error) { log.Logger.Debug("Start scan : /etc/hosts") assesses := []*types.Assessment{} @@ -23,6 +23,10 @@ func (a HostsAssessor) RequiredFiles() []string { return []string{"etc/hosts"} } +func (a HostsAssessor) RequiredExtensions() []string { + return []string{} +} + func (a HostsAssessor) RequiredPermissions() []os.FileMode { return []os.FileMode{} } diff --git a/pkg/assessor/manifest/manifest.go b/pkg/assessor/manifest/manifest.go index 3f283e7..71a4cfd 100644 --- a/pkg/assessor/manifest/manifest.go +++ b/pkg/assessor/manifest/manifest.go @@ -270,6 +270,10 @@ func (a ManifestAssessor) RequiredFiles() []string { return []string{} } +func (a ManifestAssessor) RequiredExtensions() []string { + return []string{} +} + func (a ManifestAssessor) RequiredPermissions() []os.FileMode { return []os.FileMode{} } diff --git a/pkg/assessor/passwd/passwd.go b/pkg/assessor/passwd/passwd.go index 1353850..39341b8 100644 --- a/pkg/assessor/passwd/passwd.go +++ b/pkg/assessor/passwd/passwd.go @@ -59,6 +59,10 @@ func (a PasswdAssessor) RequiredFiles() []string { return []string{"etc/shadow", "etc/master.passwd"} } +func (a PasswdAssessor) RequiredExtensions() []string { + return []string{} +} + func (a PasswdAssessor) RequiredPermissions() []os.FileMode { return []os.FileMode{} } diff --git a/pkg/assessor/privilege/suid.go b/pkg/assessor/privilege/suid.go index a434270..0dcfbfb 100644 --- a/pkg/assessor/privilege/suid.go +++ b/pkg/assessor/privilege/suid.go @@ -42,6 +42,10 @@ func (a PrivilegeAssessor) RequiredFiles() []string { return []string{} } +func (a PrivilegeAssessor) RequiredExtensions() []string { + return []string{} +} + //const GidMode os.FileMode = 4000 func (a PrivilegeAssessor) RequiredPermissions() []os.FileMode { return []os.FileMode{os.ModeSetgid, os.ModeSetuid} diff --git a/pkg/assessor/user/user.go b/pkg/assessor/user/user.go index b9fe58a..7f47715 100644 --- a/pkg/assessor/user/user.go +++ b/pkg/assessor/user/user.go @@ -62,6 +62,10 @@ func (a UserAssessor) RequiredFiles() []string { return []string{"etc/passwd"} } +func (a UserAssessor) RequiredExtensions() []string { + return []string{} +} + func (a UserAssessor) RequiredPermissions() []os.FileMode { return []os.FileMode{} } diff --git a/pkg/run.go b/pkg/run.go index c4c6dcc..40fb998 100644 --- a/pkg/run.go +++ b/pkg/run.go @@ -74,7 +74,8 @@ func Run(c *cli.Context) (err error) { } } manifest.AddAcceptanceKeys(c.StringSlice("accept-key")) - + scanner.AddAcceptanceFiles(c.StringSlice("accept-file")) + scanner.AddAcceptanceExtensions(c.StringSlice("accept-file-extension")) log.Logger.Debug("Start assessments...") assessments, err := scanner.ScanImage(ctx, imageName, filePath, dockerOption) if err != nil { diff --git a/pkg/scanner/scan.go b/pkg/scanner/scan.go index fafcd3d..0dbe55e 100644 --- a/pkg/scanner/scan.go +++ b/pkg/scanner/scan.go @@ -3,27 +3,38 @@ package scanner import ( "archive/tar" "context" - "flag" "os" "path/filepath" "github.com/goodwithtech/deckoder/analyzer" "github.com/goodwithtech/deckoder/extractor" "github.com/goodwithtech/deckoder/extractor/docker" - "github.com/goodwithtech/deckoder/utils" - deckodertypes "github.com/goodwithtech/deckoder/types" "github.com/goodwithtech/dockle/pkg/types" "github.com/goodwithtech/dockle/pkg/assessor" +) - "golang.org/x/crypto/ssh/terminal" +var ( + acceptanceFiles = map[string]struct{}{} + acceptanceExtensions = map[string]struct{}{} ) +func AddAcceptanceFiles(keys []string) { + for _, key := range keys { + acceptanceFiles[key] = struct{}{} + } +} + +func AddAcceptanceExtensions(keys []string) { + for _, key := range keys { + // file extension must start with . + acceptanceExtensions["."+key] = struct{}{} + } +} + func ScanImage(ctx context.Context, imageName, filePath string, dockerOption deckodertypes.DockerOption) (assessments []*types.Assessment, err error) { - var files deckodertypes.FileMap - filterFunc := createPathPermissionFilterFunc(assessor.LoadRequiredFiles(), assessor.LoadRequiredPermissions()) var ext extractor.Extractor var cleanup func() if imageName != "" { @@ -41,6 +52,8 @@ func ScanImage(ctx context.Context, imageName, filePath string, dockerOption dec } defer cleanup() ac := analyzer.New(ext) + var files deckodertypes.FileMap + filterFunc := createPathPermissionFilterFunc(assessor.LoadRequiredFiles(), assessor.LoadRequiredExtensions(), assessor.LoadRequiredPermissions()) if files, err = ac.Analyze(ctx, filterFunc); err != nil { return nil, err } @@ -49,28 +62,56 @@ func ScanImage(ctx context.Context, imageName, filePath string, dockerOption dec return assessments, nil } -func createPathPermissionFilterFunc(filenames []string, permissions []os.FileMode) deckodertypes.FilterFunc { - requiredDirNames := []string{} - requiredFileNames := []string{} +func createPathPermissionFilterFunc(filenames, extensions []string, permissions []os.FileMode) deckodertypes.FilterFunc { + requiredDirNames := map[string]struct{}{} + requiredFileNames := map[string]struct{}{} + requiredExts := map[string]struct{}{} for _, filename := range filenames { if filename[len(filename)-1] == '/' { // if filename end "/", it is directory and requiredDirNames removes last "/" - requiredDirNames = append(requiredDirNames, filepath.Clean(filename)) + requiredDirNames[filepath.Clean(filename)] = struct{}{} } else { - requiredFileNames = append(requiredFileNames, filename) + requiredFileNames[filename] = struct{}{} } } + for _, extension := range extensions { + requiredExts[extension] = struct{}{} + } return func(h *tar.Header) (bool, error) { filePath := filepath.Clean(h.Name) fileName := filepath.Base(filePath) - if utils.StringInSlice(filePath, requiredFileNames) || utils.StringInSlice(fileName, requiredFileNames) { + // Skip check if acceptance files + if _, ok := acceptanceExtensions[filepath.Ext(fileName)]; ok { + return false, nil + } + if _, ok := acceptanceFiles[filePath]; ok { + return false, nil + } + if _, ok := acceptanceFiles[fileName]; ok { + return false, nil + } + + // Check with file names + if _, ok := requiredFileNames[filePath]; ok { + return true, nil + } + if _, ok := requiredFileNames[fileName]; ok { return true, nil } + // Check with file extensions + if _, ok := requiredExts[filepath.Ext(fileName)]; ok { + return true, nil + } + + // Check with file directory name fileDir := filepath.Dir(filePath) + if _, ok := requiredDirNames[fileDir]; ok { + return true, nil + } fileDirBase := filepath.Base(fileDir) - if utils.StringInSlice(fileDir, requiredDirNames) || utils.StringInSlice(fileDirBase, requiredDirNames) { + if _, ok := requiredDirNames[fileDirBase]; ok { return true, nil } @@ -84,15 +125,3 @@ func createPathPermissionFilterFunc(filenames []string, permissions []os.FileMod return false, nil } } - -func openStream(path string) (*os.File, error) { - if path == "-" { - if terminal.IsTerminal(0) { - flag.Usage() - os.Exit(64) - } else { - return os.Stdin, nil - } - } - return os.Open(path) -} diff --git a/pkg/scanner/scan_test.go b/pkg/scanner/scan_test.go index f717b17..5cd93d4 100644 --- a/pkg/scanner/scan_test.go +++ b/pkg/scanner/scan_test.go @@ -20,6 +20,7 @@ import ( func TestScanImage(t *testing.T) { log.InitLogger(false, false) + AddAcceptanceExtensions([]string{"pem"}) testcases := map[string]struct { imageName string fileName string