From d25878205e9203bb5a7ec86f70a21177f0bdba9c Mon Sep 17 00:00:00 2001 From: AliDatadog <125997632+AliDatadog@users.noreply.github.com> Date: Wed, 29 May 2024 15:59:50 +0200 Subject: [PATCH] [CONTINT-3702] fix(sbom): Fix mount path retrieval from overlayfs (#25886) * Get mounts from overlayfs snapshotter, similarly as what is done in MountImage * prefer opts.UseMount, remove unused NoCache opts, use cache only when scanning images * move scan from snapshotter out of the default function --- pkg/sbom/collectors/containerd/containerd.go | 30 ++-- pkg/sbom/collectors/docker/docker.go | 10 +- pkg/sbom/sbom.go | 5 +- pkg/util/containerd/containerd_util.go | 72 +++++---- pkg/util/containerd/fake/containerd_util.go | 11 +- pkg/util/trivy/image.go | 5 +- pkg/util/trivy/trivy.go | 159 ++++++++++++------- pkg/util/trivy/trivy_test.go | 62 ++++++++ 8 files changed, 241 insertions(+), 113 deletions(-) create mode 100644 pkg/util/trivy/trivy_test.go diff --git a/pkg/sbom/collectors/containerd/containerd.go b/pkg/sbom/collectors/containerd/containerd.go index bc08916c9678e0..343f6c6cf83cda 100644 --- a/pkg/sbom/collectors/containerd/containerd.go +++ b/pkg/sbom/collectors/containerd/containerd.go @@ -13,6 +13,8 @@ import ( "fmt" "reflect" + "github.com/containerd/containerd" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/workloadmeta" "github.com/DataDog/datadog-agent/pkg/sbom" @@ -33,6 +35,8 @@ type scanRequest struct { imageID string } +type scannerFunc func(ctx context.Context, imgMeta *workloadmeta.ContainerImageMetadata, img containerd.Image, client cutil.ContainerdItf, scanOptions sbom.ScanOptions) (sbom.Report, error) + // NewScanRequest creates a new scan request func NewScanRequest(imageID string) sbom.ScanRequest { return scanRequest{imageID: imageID} @@ -64,8 +68,7 @@ type Collector struct { containerdClient cutil.ContainerdItf wmeta optional.Option[workloadmeta.Component] - fromFileSystem bool - closed bool + closed bool } // CleanCache cleans the cache @@ -81,7 +84,6 @@ func (c *Collector) Init(cfg config.Component, wmeta optional.Option[workloadmet } c.wmeta = wmeta c.trivyCollector = trivyCollector - c.fromFileSystem = cfg.GetBool("sbom.container_image.use_mount") c.opts = sbom.ScanOptionsFromConfig(cfg, true) return nil } @@ -117,23 +119,15 @@ func (c *Collector) Scan(ctx context.Context, request sbom.ScanRequest) sbom.Sca } var report sbom.Report - if c.fromFileSystem { - report, err = c.trivyCollector.ScanContainerdImageFromFilesystem( - ctx, - imageMeta, - image, - c.containerdClient, - c.opts, - ) + var scanner scannerFunc + if c.opts.UseMount { + scanner = c.trivyCollector.ScanContainerdImageFromFilesystem + } else if c.opts.OverlayFsScan { + scanner = c.trivyCollector.ScanContainerdImageFromSnapshotter } else { - report, err = c.trivyCollector.ScanContainerdImage( - ctx, - imageMeta, - image, - c.containerdClient, - c.opts, - ) + scanner = c.trivyCollector.ScanContainerdImage } + report, err = scanner(ctx, imageMeta, image, c.containerdClient, c.opts) scanResult := sbom.ScanResult{ Error: err, Report: report, diff --git a/pkg/sbom/collectors/docker/docker.go b/pkg/sbom/collectors/docker/docker.go index 67522efbfdcfd4..b8112919620d36 100644 --- a/pkg/sbom/collectors/docker/docker.go +++ b/pkg/sbom/collectors/docker/docker.go @@ -27,6 +27,8 @@ import ( // 1000 is already a very large default value const resultChanSize = 1000 +type scannerFunc func(ctx context.Context, imgMeta *workloadmeta.ContainerImageMetadata, client client.ImageAPIClient, scanOptions sbom.ScanOptions) (sbom.Report, error) + // scanRequest defines a scan request. This struct should be // hashable to be pushed in the work queue for processing. type scanRequest struct { @@ -106,7 +108,13 @@ func (c *Collector) Scan(ctx context.Context, request sbom.ScanRequest) sbom.Sca return sbom.ScanResult{Error: fmt.Errorf("image metadata not found for image id %s: %s", dockerScanRequest.ID(), err)} } - report, err := c.trivyCollector.ScanDockerImage( + var scanner scannerFunc + if c.opts.OverlayFsScan { + scanner = c.trivyCollector.ScanDockerImageFromGraphDriver + } else { + scanner = c.trivyCollector.ScanDockerImage + } + report, err := scanner( ctx, imageMeta, c.cl, diff --git a/pkg/sbom/sbom.go b/pkg/sbom/sbom.go index d1bf15e51dd805..2a0d7058b323e5 100644 --- a/pkg/sbom/sbom.go +++ b/pkg/sbom/sbom.go @@ -34,9 +34,9 @@ type ScanOptions struct { Timeout time.Duration WaitAfter time.Duration Fast bool - NoCache bool // Caching doesn't really provide any value when scanning filesystem as the filesystem has to be walked to compute the keys CollectFiles bool UseMount bool + OverlayFsScan bool } // ScanOptionsFromConfig loads the scanning options from the configuration @@ -48,8 +48,7 @@ func ScanOptionsFromConfig(cfg config.Component, containers bool) (scanOpts Scan scanOpts.WaitAfter = time.Duration(cfg.GetInt("sbom.container_image.scan_interval")) * time.Second scanOpts.Analyzers = cfg.GetStringSlice("sbom.container_image.analyzers") scanOpts.UseMount = cfg.GetBool("sbom.container_image.use_mount") - } else { - scanOpts.NoCache = true + scanOpts.OverlayFsScan = cfg.GetBool("sbom.container_image.overlayfs_direct_scan") } if len(scanOpts.Analyzers) == 0 { diff --git a/pkg/util/containerd/containerd_util.go b/pkg/util/containerd/containerd_util.go index 0e0d57c1b9592f..1b9b043b092683 100644 --- a/pkg/util/containerd/containerd_util.go +++ b/pkg/util/containerd/containerd_util.go @@ -16,11 +16,12 @@ import ( "strings" "time" + "github.com/opencontainers/image-spec/identity" + "github.com/DataDog/datadog-agent/pkg/config" dderrors "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/util/retry" - "github.com/opencontainers/image-spec/identity" "github.com/containerd/containerd" "github.com/containerd/containerd/api/types" @@ -70,6 +71,7 @@ type ContainerdItf interface { CallWithClientContext(namespace string, f func(context.Context) error) error IsSandbox(namespace string, ctn containerd.Container) (bool, error) MountImage(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image, targetDir string) (func(context.Context) error, error) + Mounts(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image) ([]mount.Mount, error) } // ContainerdUtil is the util used to interact with the Containerd api. @@ -384,28 +386,28 @@ func (c *ContainerdUtil) IsSandbox(namespace string, ctn containerd.Container) ( return labels["io.cri-containerd.kind"] == "sandbox", nil } -// MountImage mounts the given image in the targetDir specified -func (c *ContainerdUtil) MountImage(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image, targetDir string) (func(context.Context) error, error) { +// getMounts retrieves mounts and returns a function to clean the snapshot and release the lease. The lease is already released in error cases. +func (c *ContainerdUtil) getMounts(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image) ([]mount.Mount, func(context.Context) error, error) { snapshotter := containerd.DefaultSnapshotter ctx = namespaces.WithNamespace(ctx, namespace) // Checking if image is already unpacked imgUnpacked, err := img.IsUnpacked(ctx, snapshotter) if err != nil { - return nil, fmt.Errorf("unable to check if image named: %s is unpacked, err: %w", img.Name(), err) + return nil, nil, fmt.Errorf("unable to check if image named: %s is unpacked, err: %w", img.Name(), err) } if !imgUnpacked { - return nil, fmt.Errorf("unable to scan image named: %s, image is not unpacked", img.Name()) + return nil, nil, fmt.Errorf("unable to scan image named: %s, image is not unpacked", img.Name()) } // Getting image id imgConfig, err := img.Config(ctx) if err != nil { - return nil, fmt.Errorf("unable to get image config for image named: %s, err: %w", img.Name(), err) + return nil, nil, fmt.Errorf("unable to get image config for image named: %s, err: %w", img.Name(), err) } imageID := imgConfig.Digest.String() - // Adding a lease to cleanup dandling snaphots at expiration + // Adding a lease to cleanup dangling snapshots at expiration ctx, done, err := c.cl.WithLease(ctx, leases.WithID(imageID), leases.WithExpiration(expiration), @@ -414,7 +416,7 @@ func (c *ContainerdUtil) MountImage(ctx context.Context, expiration time.Duratio }), ) if err != nil && !errdefs.IsAlreadyExists(err) { - return nil, fmt.Errorf("unable to get a lease, err: %w", err) + return nil, nil, fmt.Errorf("unable to get a lease, err: %w", err) } // Getting top layer image id @@ -423,18 +425,17 @@ func (c *ContainerdUtil) MountImage(ctx context.Context, expiration time.Duratio if err := done(ctx); err != nil { log.Warnf("Unable to cancel containerd lease with id: %s, err: %v", imageID, err) } - return nil, fmt.Errorf("unable to get layers digests for image: %s, err: %w", imageID, err) + return nil, nil, fmt.Errorf("unable to get layers digests for image: %s, err: %w", imageID, err) } chainID := identity.ChainID(diffIDs).String() - - // Creating snaphot for the top layer + // Creating snapshot for the top layer s := c.cl.SnapshotService(snapshotter) mounts, err := s.View(ctx, imageID, chainID) if err != nil && !errdefs.IsAlreadyExists(err) { if err := done(ctx); err != nil { log.Warnf("Unable to cancel containerd lease with id: %s, err: %v", imageID, err) } - return nil, fmt.Errorf("unable to build snapshot for image: %s, err: %w", imageID, err) + return nil, nil, fmt.Errorf("unable to build snapshot for image: %s, err: %w", imageID, err) } cleanSnapshot := func(ctx context.Context) error { return s.Remove(ctx, imageID) @@ -448,7 +449,7 @@ func (c *ContainerdUtil) MountImage(ctx context.Context, expiration time.Duratio if err := done(ctx); err != nil { log.Warnf("Unable to cancel containerd lease with id: %s, err: %v", imageID, err) } - return nil, fmt.Errorf("No snapshots returned for image: %s, err: %w", imageID, err) + return nil, nil, fmt.Errorf("No snapshots returned for image: %s", imageID) } // Transforming mounts in case we're running in a container @@ -460,32 +461,47 @@ func (c *ContainerdUtil) MountImage(ctx context.Context, expiration time.Duratio } } } - - // Mouting returned mounts - log.Infof("Mounting %+v to %s", mounts, targetDir) - if err := mount.All(mounts, targetDir); err != nil { + return mounts, func(ctx context.Context) error { + ctx = namespaces.WithNamespace(ctx, namespace) if err := cleanSnapshot(ctx); err != nil { log.Warnf("Unable to clean snapshot with id: %s, err: %v", imageID, err) } if err := done(ctx); err != nil { log.Warnf("Unable to cancel containerd lease with id: %s, err: %v", imageID, err) } - return nil, fmt.Errorf("unable to mount image %s to dir %s, err: %w", imageID, targetDir, err) + return nil + }, nil +} + +// Mounts returns the mounts for an image +func (c *ContainerdUtil) Mounts(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image) ([]mount.Mount, error) { + mounts, clean, err := c.getMounts(ctx, expiration, namespace, img) + if err != nil { + return nil, err + } + if err := clean(ctx); err != nil { + return nil, fmt.Errorf("unable to clean snapshot, err: %w", err) } + return mounts, nil +} +// MountImage mounts an image to a directory +func (c *ContainerdUtil) MountImage(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image, targetDir string) (func(context.Context) error, error) { + mounts, clean, err := c.getMounts(ctx, expiration, namespace, img) + if err != nil { + return nil, err + } + if err := mount.All(mounts, targetDir); err != nil { + if err := clean(ctx); err != nil { + log.Warnf("Unable to clean snapshot, err: %v", err) + } + return nil, fmt.Errorf("unable to mount image %s to dir %s, err: %w", img.Name(), targetDir, err) + } return func(ctx context.Context) error { ctx = namespaces.WithNamespace(ctx, namespace) - if err := mount.UnmountAll(targetDir, 0); err != nil { - return fmt.Errorf("unable to unmount directory: %s for image: %s, err: %w", targetDir, imageID, err) + return fmt.Errorf("unable to unmount directory: %s for image: %s, err: %w", targetDir, img.Name(), err) } - if err := cleanSnapshot(ctx); err != nil && !errdefs.IsNotFound(err) { - return fmt.Errorf("unable to cleanup snapshot for image: %s, err: %w", imageID, err) - } - if err := done(ctx); err != nil { - return fmt.Errorf("unable to cancel lease for image: %s, err: %w", imageID, err) - } - - return nil + return clean(ctx) }, nil } diff --git a/pkg/util/containerd/fake/containerd_util.go b/pkg/util/containerd/fake/containerd_util.go index 9a3446000e39c1..36a92d249e851c 100644 --- a/pkg/util/containerd/fake/containerd_util.go +++ b/pkg/util/containerd/fake/containerd_util.go @@ -15,6 +15,7 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/mount" "github.com/containerd/containerd/oci" "github.com/DataDog/datadog-agent/pkg/util/retry" @@ -47,6 +48,7 @@ type MockedContainerdClient struct { MockCallWithClientContext func(namespace string, f func(context.Context) error) error MockIsSandbox func(namespace string, ctn containerd.Container) (bool, error) MockMountImage func(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image, targetDir string) (func(context.Context) error, error) + MockMounts func(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image) ([]mount.Mount, error) } // Close is a mock method @@ -145,9 +147,7 @@ func (client *MockedContainerdClient) GetEvents() containerd.EventService { } // Spec is a mock method -// -//nolint:revive // TODO(CINT) Fix revive linter -func (client *MockedContainerdClient) Spec(namespace string, ctn containers.Container, maxSize int) (*oci.Spec, error) { +func (client *MockedContainerdClient) Spec(namespace string, ctn containers.Container, _ int) (*oci.Spec, error) { return client.MockSpec(namespace, ctn) } @@ -170,3 +170,8 @@ func (client *MockedContainerdClient) IsSandbox(namespace string, ctn containerd func (client *MockedContainerdClient) MountImage(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image, targetDir string) (func(context.Context) error, error) { return client.MockMountImage(ctx, expiration, namespace, img, targetDir) } + +// Mounts is a mock method +func (client *MockedContainerdClient) Mounts(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image) ([]mount.Mount, error) { + return client.MockMounts(ctx, expiration, namespace, img) +} diff --git a/pkg/util/trivy/image.go b/pkg/util/trivy/image.go index 4340dc156f5ec7..f4043837e528dd 100644 --- a/pkg/util/trivy/image.go +++ b/pkg/util/trivy/image.go @@ -17,7 +17,6 @@ import ( "sync" "time" - "github.com/DataDog/datadog-agent/pkg/sbom/telemetry" fimage "github.com/aquasecurity/trivy/pkg/fanal/image" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -26,6 +25,8 @@ import ( "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/samber/lo" "golang.org/x/xerrors" + + "github.com/DataDog/datadog-agent/pkg/sbom/telemetry" ) var mu sync.Mutex @@ -62,7 +63,7 @@ func imageOpener(ctx context.Context, collector, ref string, f *os.File, imageSa // image is a wrapper for github.com/google/go-containerregistry/pkg/v1/daemon.Image // daemon.Image loads the entire image into the memory at first, -// but it doesn't need to load it if the information is already in the cache, +// but it doesn't need to load it if the information is already in the persistentCache, // To avoid entire loading, this wrapper uses ImageInspectWithRaw and checks image ID and layer IDs. type image struct { v1.Image diff --git a/pkg/util/trivy/trivy.go b/pkg/util/trivy/trivy.go index e1e54ef17e589a..d56824e20627bc 100644 --- a/pkg/util/trivy/trivy.go +++ b/pkg/util/trivy/trivy.go @@ -20,6 +20,9 @@ import ( "sync" "time" + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/namespaces" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/workloadmeta" "github.com/DataDog/datadog-agent/pkg/sbom" @@ -77,7 +80,7 @@ type collectorConfig struct { type Collector struct { config collectorConfig cacheInitialized sync.Once - cache CacheWithCleaner + persistentCache CacheWithCleaner osScanner ospkg.Scanner langScanner langpkg.Scanner vulnClient vulnerability.Client @@ -171,7 +174,7 @@ func NewCollector(cfg config.Component, wmeta optional.Option[workloadmeta.Compo return &Collector{ config: collectorConfig{ clearCacheOnClose: cfg.GetBool("sbom.clear_cache_on_exit"), - maxCacheSize: cfg.GetInt("sbom.cache.max_disk_size"), + maxCacheSize: cfg.GetInt("sbom.persistentCache.max_disk_size"), overlayFSSupport: cfg.GetBool("sbom.container_image.overlayfs_direct_scan"), }, osScanner: ospkg.NewScanner(), @@ -199,33 +202,33 @@ func GetGlobalCollector(cfg config.Component, wmeta optional.Option[workloadmeta // Close closes the collector func (c *Collector) Close() error { - if c.cache == nil { + if c.persistentCache == nil { return nil } if c.config.clearCacheOnClose { - if err := c.cache.Clear(); err != nil { - return fmt.Errorf("error when clearing trivy cache: %w", err) + if err := c.persistentCache.Clear(); err != nil { + return fmt.Errorf("error when clearing trivy persistentCache: %w", err) } } - return c.cache.Close() + return c.persistentCache.Close() } -// CleanCache cleans the cache +// CleanCache cleans the persistentCache func (c *Collector) CleanCache() error { - if c.cache != nil { - return c.cache.clean() + if c.persistentCache != nil { + return c.persistentCache.clean() } return nil } -// getCache returns the cache with the cache Cleaner. It should initializes the cache +// getCache returns the persistentCache with the persistentCache Cleaner. It should initializes the persistentCache // only once to avoid blocking the CLI with the `flock` file system. func (c *Collector) getCache() (CacheWithCleaner, error) { var err error c.cacheInitialized.Do(func() { - c.cache, err = NewCustomBoltCache( + c.persistentCache, err = NewCustomBoltCache( c.wmeta, defaultCacheDir(), c.config.maxCacheSize, @@ -236,11 +239,11 @@ func (c *Collector) getCache() (CacheWithCleaner, error) { return nil, err } - return c.cache, nil + return c.persistentCache, nil } -// ScanDockerImage scans a docker image -func (c *Collector) ScanDockerImage(ctx context.Context, imgMeta *workloadmeta.ContainerImageMetadata, client client.ImageAPIClient, scanOptions sbom.ScanOptions) (sbom.Report, error) { +// ScanDockerImageFromGraphDriver scans a docker image directly from the graph driver +func (c *Collector) ScanDockerImageFromGraphDriver(ctx context.Context, imgMeta *workloadmeta.ContainerImageMetadata, client client.ImageAPIClient, scanOptions sbom.ScanOptions) (sbom.Report, error) { fanalImage, cleanup, err := convertDockerImage(ctx, client, imgMeta) if cleanup != nil { defer cleanup() @@ -250,25 +253,38 @@ func (c *Collector) ScanDockerImage(ctx context.Context, imgMeta *workloadmeta.C return nil, fmt.Errorf("unable to convert docker image, err: %w", err) } - if c.config.overlayFSSupport && fanalImage.inspect.GraphDriver.Name == "overlay2" { - return c.scanOverlayFS(ctx, fanalImage, imgMeta, scanOptions) + if fanalImage.inspect.GraphDriver.Name == "overlay2" { + var layers []string + if layerDirs, ok := fanalImage.inspect.GraphDriver.Data["LowerDir"]; ok { + layers = append(layers, strings.Split(layerDirs, ":")...) + } + + if layerDirs, ok := fanalImage.inspect.GraphDriver.Data["UpperDir"]; ok { + layers = append(layers, strings.Split(layerDirs, ":")...) + } + return c.scanOverlayFS(ctx, layers, imgMeta, scanOptions) } - return c.scanImage(ctx, fanalImage, imgMeta, scanOptions) + return nil, fmt.Errorf("unsupported graph driver: %s", fanalImage.inspect.GraphDriver.Name) } -func (c *Collector) scanOverlayFS(ctx context.Context, fanalImage *image, imgMeta *workloadmeta.ContainerImageMetadata, scanOptions sbom.ScanOptions) (sbom.Report, error) { - var layers []string - if layerDirs, ok := fanalImage.inspect.GraphDriver.Data["LowerDir"]; ok { - layers = append(layers, strings.Split(layerDirs, ":")...) +// ScanDockerImage scans a docker image by exporting it and scanning the tarball +func (c *Collector) ScanDockerImage(ctx context.Context, imgMeta *workloadmeta.ContainerImageMetadata, client client.ImageAPIClient, scanOptions sbom.ScanOptions) (sbom.Report, error) { + fanalImage, cleanup, err := convertDockerImage(ctx, client, imgMeta) + if cleanup != nil { + defer cleanup() } - if layerDirs, ok := fanalImage.inspect.GraphDriver.Data["UpperDir"]; ok { - layers = append(layers, strings.Split(layerDirs, ":")...) + if err != nil { + return nil, fmt.Errorf("unable to convert docker image, err: %w", err) } - fs := NewFS(layers) - report, err := c.scanFilesystem(ctx, fs, ".", imgMeta, scanOptions) + return c.scanImage(ctx, fanalImage, imgMeta, scanOptions) +} + +func (c *Collector) scanOverlayFS(ctx context.Context, layers []string, imgMeta *workloadmeta.ContainerImageMetadata, scanOptions sbom.ScanOptions) (sbom.Report, error) { + overlayFsReader := NewFS(layers) + report, err := c.scanFilesystem(ctx, overlayFsReader, ".", imgMeta, scanOptions) if err != nil { return nil, err } @@ -276,41 +292,53 @@ func (c *Collector) scanOverlayFS(ctx context.Context, fanalImage *image, imgMet return report, nil } -// ScanContainerdImage scans containerd image -func (c *Collector) ScanContainerdImage(ctx context.Context, imgMeta *workloadmeta.ContainerImageMetadata, img containerd.Image, client cutil.ContainerdItf, scanOptions sbom.ScanOptions) (sbom.Report, error) { - fanalImage, cleanup, err := convertContainerdImage(ctx, client.RawClient(), imgMeta, img) - if cleanup != nil { - defer cleanup() - } +// ScanContainerdImageFromSnapshotter scans containerd image directly from the snapshotter +func (c *Collector) ScanContainerdImageFromSnapshotter(ctx context.Context, imgMeta *workloadmeta.ContainerImageMetadata, img containerd.Image, client cutil.ContainerdItf, scanOptions sbom.ScanOptions) (sbom.Report, error) { + // Computing duration of containerd lease + deadline, _ := ctx.Deadline() + expiration := deadline.Sub(time.Now().Add(cleanupTimeout)) + clClient := client.RawClient() + imageID := imgMeta.ID + + mounts, err := client.Mounts(ctx, expiration, imgMeta.Namespace, img) if err != nil { - return nil, fmt.Errorf("unable to convert containerd image, err: %w", err) + return nil, fmt.Errorf("unable to get mounts for image %s, err: %w", imgMeta.ID, err) + } + layers := extractLayersFromOverlayFSMounts(mounts) + if len(layers) == 0 { + return nil, fmt.Errorf("unable to extract layers from overlayfs mounts for image %s", imgMeta.ID) } - if c.config.overlayFSSupport && fanalImage.inspect.GraphDriver.Name == "overlay2" { - // Computing duration of containerd lease - deadline, _ := ctx.Deadline() - expiration := deadline.Sub(time.Now().Add(cleanupTimeout)) - - clClient := client.RawClient() - - imageID := imgMeta.ID + ctx = namespaces.WithNamespace(ctx, imgMeta.Namespace) + // Adding a lease to cleanup dandling snaphots at expiration + ctx, done, err := clClient.WithLease(ctx, + leases.WithID(imageID), + leases.WithExpiration(expiration), + leases.WithLabels(map[string]string{ + "containerd.io/gc.ref.snapshot." + containerd.DefaultSnapshotter: imageID, + }), + ) + if err != nil && !errdefs.IsAlreadyExists(err) { + return nil, fmt.Errorf("unable to get a lease, err: %w", err) + } - // Adding a lease to cleanup dandling snaphots at expiration - ctx, done, err := clClient.WithLease(ctx, - leases.WithID(imageID), - leases.WithExpiration(expiration), - ) - if err != nil && !errdefs.IsAlreadyExists(err) { - return nil, fmt.Errorf("unable to get a lease, err: %w", err) - } + report, err := c.scanOverlayFS(ctx, layers, imgMeta, scanOptions) - report, err := c.scanOverlayFS(ctx, fanalImage, imgMeta, scanOptions) + if err := done(ctx); err != nil { + log.Warnf("Unable to cancel containerd lease with id: %s, err: %v", imageID, err) + } - if err := done(ctx); err != nil { - log.Warnf("Unable to cancel containerd lease with id: %s, err: %v", imageID, err) - } + return report, err +} - return report, err +// ScanContainerdImage scans containerd image by exporting it and scanning the tarball +func (c *Collector) ScanContainerdImage(ctx context.Context, imgMeta *workloadmeta.ContainerImageMetadata, img containerd.Image, client cutil.ContainerdItf, scanOptions sbom.ScanOptions) (sbom.Report, error) { + fanalImage, cleanup, err := convertContainerdImage(ctx, client.RawClient(), imgMeta, img) + if cleanup != nil { + defer cleanup() + } + if err != nil { + return nil, fmt.Errorf("unable to convert containerd image, err: %w", err) } return c.scanImage(ctx, fanalImage, imgMeta, scanOptions) @@ -352,7 +380,7 @@ func (c *Collector) ScanContainerdImageFromFilesystem(ctx context.Context, imgMe } func (c *Collector) scanFilesystem(ctx context.Context, fsys fs.FS, path string, imgMeta *workloadmeta.ContainerImageMetadata, scanOptions sbom.ScanOptions) (sbom.Report, error) { - // For filesystem scans, it is required to walk the filesystem to get the cache key so caching does not add any value. + // For filesystem scans, it is required to walk the filesystem to get the persistentCache key so caching does not add any value. // TODO: Cache directly the trivy report for container images cache := newMemoryCache() @@ -361,7 +389,7 @@ func (c *Collector) scanFilesystem(ctx context.Context, fsys fs.FS, path string, return nil, fmt.Errorf("unable to create artifact from fs, err: %w", err) } - trivyReport, err := c.scan(ctx, fsArtifact, applier.NewApplier(cache), imgMeta, cache) + trivyReport, err := c.scan(ctx, fsArtifact, applier.NewApplier(cache), imgMeta, cache, false) if err != nil { if imgMeta != nil { return nil, fmt.Errorf("unable to marshal report to sbom format for image %s, err: %w", imgMeta.ID, err) @@ -388,8 +416,8 @@ func (c *Collector) ScanFilesystem(ctx context.Context, fsys fs.FS, path string, return c.scanFilesystem(ctx, fsys, path, nil, scanOptions) } -func (c *Collector) scan(ctx context.Context, artifact artifact.Artifact, applier applier.Applier, imgMeta *workloadmeta.ContainerImageMetadata, cache CacheWithCleaner) (*types.Report, error) { - if imgMeta != nil && cache != nil { +func (c *Collector) scan(ctx context.Context, artifact artifact.Artifact, applier applier.Applier, imgMeta *workloadmeta.ContainerImageMetadata, cache CacheWithCleaner, useCache bool) (*types.Report, error) { + if useCache && imgMeta != nil && cache != nil { // The artifact reference is only needed to clean up the blobs after the scan. // It is re-generated from cached partial results during the scan. artifactReference, err := artifact.Inspect(ctx) @@ -423,7 +451,7 @@ func (c *Collector) scanImage(ctx context.Context, fanalImage ftypes.Image, imgM return nil, fmt.Errorf("unable to create artifact from image, err: %w", err) } - trivyReport, err := c.scan(ctx, imageArtifact, applier.NewApplier(cache), imgMeta, c.cache) + trivyReport, err := c.scan(ctx, imageArtifact, applier.NewApplier(cache), imgMeta, c.persistentCache, true) if err != nil { return nil, fmt.Errorf("unable to marshal report to sbom format, err: %w", err) } @@ -434,3 +462,18 @@ func (c *Collector) scanImage(ctx context.Context, fanalImage ftypes.Image, imgM marshaler: c.marshaler, }, nil } + +func extractLayersFromOverlayFSMounts(mounts []mount.Mount) []string { + var layers []string + for _, mount := range mounts { + for _, opt := range mount.Options { + for _, prefix := range []string{"upperdir=", "lowerdir="} { + trimmedOpt := strings.TrimPrefix(opt, prefix) + if trimmedOpt != opt { + layers = append(layers, strings.Split(trimmedOpt, ":")...) + } + } + } + } + return layers +} diff --git a/pkg/util/trivy/trivy_test.go b/pkg/util/trivy/trivy_test.go new file mode 100644 index 00000000000000..20bd729cde6c59 --- /dev/null +++ b/pkg/util/trivy/trivy_test.go @@ -0,0 +1,62 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build trivy + +// Package trivy holds the scan components +package trivy + +import ( + "testing" + + "github.com/containerd/containerd/mount" + "github.com/stretchr/testify/assert" +) + +// TestExtractLayersFromOverlayFSMounts checks if the function correctly extracts layer paths from Mount options. +func TestExtractLayersFromOverlayFSMounts(t *testing.T) { + for _, tt := range []struct { + name string + mounts []mount.Mount + want []string + }{ + { + name: "No mounts", + mounts: []mount.Mount{}, + }, + { + name: "Single upperdir", + mounts: []mount.Mount{{Options: []string{"someoption=somevalue", "upperdir=/path/to/upper"}}}, + want: []string{"/path/to/upper"}, + }, + { + name: "Single lowerdir", + mounts: []mount.Mount{{Options: []string{"someoption=somevalue", "lowerdir=/path/to/lower"}}}, + want: []string{"/path/to/lower"}, + }, + { + name: "Multiple lowerdir", + mounts: []mount.Mount{{Options: []string{"someoption=somevalue", "lowerdir=/path/to/lower1:/path/to/lower2"}}}, + want: []string{"/path/to/lower1", "/path/to/lower2"}, + }, + { + name: "Multiple options", + mounts: []mount.Mount{{Options: []string{"someoption=somevalue", "upperdir=/path/to/upper", "lowerdir=/path/to/lower1:/path/to/lower2"}}}, + want: []string{"/path/to/upper", "/path/to/lower1", "/path/to/lower2"}, + }, + { + name: "Multiple mounts", + mounts: []mount.Mount{ + {Options: []string{"someoption=somevalue", "upperdir=/path/to/upper1"}}, + {Options: []string{"someoption=somevalue", "lowerdir=/path/to/lower1:/path/to/lower2"}}, + }, + want: []string{"/path/to/upper1", "/path/to/lower1", "/path/to/lower2"}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, extractLayersFromOverlayFSMounts(tt.mounts)) + }) + } +}