Issue 966: Extend HasSBOM to include references to included software … (

#1367) * Issue 966: Extend HasSBOM to include references to included software (Package and Artifact), dependencies and occurrences Update package and source ingestion to return all relevant IDs to caller. Include includes in SBOM filtering Signed-off-by: Kevin Conner <kev.conner@gmail.com> * Sort versions based on ID to guarantee order Signed-off-by: Kevin Conner <kev.conner@gmail.com> --------- Signed-off-by: Kevin Conner <kev.conner@gmail.com>
guacsec · Oct 26, 2023 · 165897d · 165897d
1 parent 4e01d67
commit 165897d
Show file tree

Hide file tree

Showing 45 changed files with 5,710 additions and 341 deletions.
diff --git a/internal/testing/e2e/e2e b/internal/testing/e2e/e2e
@@ -69,7 +69,7 @@ diff -u "${SCRIPT_DIR}/expectPkgQ1.json" "${GUAC_DIR}/gotPkgQ1.json"
 cat "$queries" | gql-cli http://localhost:8080/query -o PkgQ2 | jq ' .packages[].namespaces |= sort ' > "${GUAC_DIR}/gotPkgQ2.json"
 diff -u "${SCRIPT_DIR}/expectPkgQ2.json" "${GUAC_DIR}/gotPkgQ2.json"
 
-cat "$queries" | gql-cli http://localhost:8080/query -o PkgQ3 | jq ' del(.. | .id?) | .packages[].namespaces |= sort_by(.namespace) | .packages[].namespaces[].names[].versions |= sort_by(.version) | .packages[].namespaces[].names[].versions[].qualifiers |= sort_by(.key) ' > "${GUAC_DIR}/gotPkgQ3.json"
+cat "$queries" | gql-cli http://localhost:8080/query -o PkgQ3 | jq ' .packages[].namespaces |= sort_by(.namespace) | .packages[].namespaces[].names[].versions |= sort_by(.id) | .packages[].namespaces[].names[].versions[].qualifiers |= sort_by(.key) | del(.. | .id?) ' > "${GUAC_DIR}/gotPkgQ3.json"
 diff -u "${SCRIPT_DIR}/expectPkgQ3.json" "${GUAC_DIR}/gotPkgQ3.json"
 
 cat "$queries" | gql-cli http://localhost:8080/query -o PkgQ4 | jq ' del(.. | .id?) '> "${GUAC_DIR}/gotPkgQ4.json"

diff --git a/internal/testing/mocks/backend.go b/internal/testing/mocks/backend.go
diff --git a/pkg/assembler/assembler.go b/pkg/assembler/assembler.go
@@ -147,7 +147,8 @@ type HasSBOMIngest struct {
  Pkg *generated.PkgInputSpec `json:"pkg,omitempty"`
  Artifact *generated.ArtifactInputSpec `json:"artifact,omitempty"`
 
- HasSBOM *generated.HasSBOMInputSpec `json:"hasSbom,omitempty"`
+ HasSBOM *generated.HasSBOMInputSpec `json:"hasSbom,omitempty"`
+ Includes *generated.HasSBOMIncludesInputSpec `json:"includes,omitempty"`
 }
 
 type VexIngest struct {

diff --git a/pkg/assembler/backends/arangodb/hasSBOM.go b/pkg/assembler/backends/arangodb/hasSBOM.go
@@ -218,7 +218,8 @@ func getHasSBOMQueryValues(pkg *model.PkgInputSpec, artifact *model.ArtifactInpu
  return values
 }
 
-func (c *arangoClient) IngestHasSBOMs(ctx context.Context, subjects model.PackageOrArtifactInputs, hasSBOMs []*model.HasSBOMInputSpec) ([]*model.HasSbom, error) {
+func (c *arangoClient) IngestHasSBOMs(ctx context.Context, subjects model.PackageOrArtifactInputs, hasSBOMs []*model.HasSBOMInputSpec, includes []*model.HasSBOMIncludesInputSpec) ([]*model.HasSbom, error) {
+ // TODO(knrc) - handle includes
  if len(subjects.Packages) > 0 {
  var listOfValues []map[string]any
 
@@ -401,7 +402,8 @@ func (c *arangoClient) IngestHasSBOMs(ctx context.Context, subjects model.Packag
  }
 }
 
-func (c *arangoClient) IngestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, hasSbom model.HasSBOMInputSpec) (*model.HasSbom, error) {
+func (c *arangoClient) IngestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, hasSbom model.HasSBOMInputSpec, includes model.HasSBOMIncludesInputSpec) (*model.HasSbom, error) {
+ // TODO(knrc) - handle includes
  if subject.Artifact != nil {
  query := `LET artifact = FIRST(FOR art IN artifacts FILTER art.algorithm == @art_algorithm FILTER art.digest == @art_digest RETURN art)
  

diff --git a/pkg/assembler/backends/arangodb/hasSBOM_test.go b/pkg/assembler/backends/arangodb/hasSBOM_test.go
@@ -45,6 +45,7 @@ func TestHasSBOM(t *testing.T) {
  type call struct {
  Sub model.PackageOrArtifactInput
  HS *model.HasSBOMInputSpec
+ Inc *model.HasSBOMIncludesInputSpec
  }
  tests := []struct {
  Name string
@@ -577,7 +578,8 @@ func TestHasSBOM(t *testing.T) {
  }
  }
  for _, o := range test.Calls {
- found, err := b.IngestHasSbom(ctx, o.Sub, *o.HS)
+ // TODO (knrc) handle includes
+ found, err := b.IngestHasSbom(ctx, o.Sub, *o.HS, model.HasSBOMIncludesInputSpec{})
  if (err != nil) != test.ExpIngestErr {
  t.Fatalf("did not get expected ingest error, want: %v, got: %v", test.ExpIngestErr, err)
  }
@@ -640,6 +642,7 @@ func TestIngestHasSBOM(t *testing.T) {
  type call struct {
  Sub model.PackageOrArtifactInputs
  HS []*model.HasSBOMInputSpec
+ Inc []*model.HasSBOMIncludesInputSpec
  }
  tests := []struct {
  Name string
@@ -835,7 +838,7 @@ func TestIngestHasSBOM(t *testing.T) {
  }
  }
  for _, o := range test.Calls {
- _, err := b.IngestHasSBOMs(ctx, o.Sub, o.HS)
+ _, err := b.IngestHasSBOMs(ctx, o.Sub, o.HS, o.Inc)
  if (err != nil) != test.ExpIngestErr {
  t.Fatalf("did not get expected ingest error, want: %v, got: %v", test.ExpIngestErr, err)
  }
@@ -1031,7 +1034,8 @@ func Test_buildHasSbomByID(t *testing.T) {
  }
  }
  for _, o := range test.Calls {
- found, err := b.IngestHasSbom(ctx, o.Sub, *o.HS)
+ // TODO (knrc) handle includes
+ found, err := b.IngestHasSbom(ctx, o.Sub, *o.HS, model.HasSBOMIncludesInputSpec{})
  if (err != nil) != test.ExpIngestErr {
  t.Fatalf("did not get expected ingest error, want: %v, got: %v", test.ExpIngestErr, err)
  }

diff --git a/pkg/assembler/backends/arangodb/path_test.go b/pkg/assembler/backends/arangodb/path_test.go
@@ -883,7 +883,8 @@ func Test_Nodes(t *testing.T) {
  nodeID = found.ID
  }
  if tt.hasSBOMCall != nil {
- found, err := b.IngestHasSbom(ctx, tt.hasSBOMCall.Sub, *tt.hasSBOMCall.HS)
+ // TODO (knrc) handle includes
+ found, err := b.IngestHasSbom(ctx, tt.hasSBOMCall.Sub, *tt.hasSBOMCall.HS, model.HasSBOMIncludesInputSpec{})
  if (err != nil) != tt.wantErr {
  t.Fatalf("did not get expected ingest error, want: %v, got: %v", tt.wantErr, err)
  }
@@ -3377,7 +3378,8 @@ func Test_Neighbors(t *testing.T) {
  }
  }
  if tt.hasSBOMCall != nil {
- found, err := b.IngestHasSbom(ctx, tt.hasSBOMCall.Sub, *tt.hasSBOMCall.HS)
+ // TODO (knrc) handle includes
+ found, err := b.IngestHasSbom(ctx, tt.hasSBOMCall.Sub, *tt.hasSBOMCall.HS, model.HasSBOMIncludesInputSpec{})
  if (err != nil) != tt.wantErr {
  t.Fatalf("did not get expected ingest error, want: %v, got: %v", tt.wantErr, err)
  }

diff --git a/pkg/assembler/backends/backends.go b/pkg/assembler/backends/backends.go
@@ -77,8 +77,8 @@ type Backend interface {
  IngestCertifyLegals(ctx context.Context, subjects model.PackageOrSourceInputs, declaredLicensesList [][]*model.LicenseInputSpec, discoveredLicensesList [][]*model.LicenseInputSpec, certifyLegals []*model.CertifyLegalInputSpec) ([]*model.CertifyLegal, error)
  IngestDependency(ctx context.Context, pkg model.PkgInputSpec, depPkg model.PkgInputSpec, depPkgMatchType model.MatchFlags, dependency model.IsDependencyInputSpec) (*model.IsDependency, error)
  IngestDependencies(ctx context.Context, pkgs []*model.PkgInputSpec, depPkgs []*model.PkgInputSpec, depPkgMatchType model.MatchFlags, dependencies []*model.IsDependencyInputSpec) ([]*model.IsDependency, error)
- IngestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, hasSbom model.HasSBOMInputSpec) (*model.HasSbom, error)
- IngestHasSBOMs(ctx context.Context, subjects model.PackageOrArtifactInputs, hasSBOMs []*model.HasSBOMInputSpec) ([]*model.HasSbom, error)
+ IngestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, hasSbom model.HasSBOMInputSpec, includes model.HasSBOMIncludesInputSpec) (*model.HasSbom, error)
+ IngestHasSBOMs(ctx context.Context, subjects model.PackageOrArtifactInputs, hasSBOMs []*model.HasSBOMInputSpec, includes []*model.HasSBOMIncludesInputSpec) ([]*model.HasSbom, error)
  IngestHasSourceAt(ctx context.Context, pkg model.PkgInputSpec, pkgMatchType model.MatchFlags, source model.SourceInputSpec, hasSourceAt model.HasSourceAtInputSpec) (*model.HasSourceAt, error)
  IngestHasSourceAts(ctx context.Context, pkgs []*model.PkgInputSpec, pkgMatchType *model.MatchFlags, sources []*model.SourceInputSpec, hasSourceAts []*model.HasSourceAtInputSpec) ([]string, error)
  IngestHasMetadata(ctx context.Context, subject model.PackageSourceOrArtifactInput, pkgMatchType *model.MatchFlags, hasMetadata model.HasMetadataInputSpec) (*model.HasMetadata, error)

diff --git a/pkg/assembler/backends/ent/backend/sbom.go b/pkg/assembler/backends/ent/backend/sbom.go
@@ -69,7 +69,8 @@ func (b *EntBackend) HasSBOM(ctx context.Context, spec *model.HasSBOMSpec) ([]*m
  return collect(records, toModelHasSBOM), nil
 }
 
-func (b *EntBackend) IngestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, spec model.HasSBOMInputSpec) (*model.HasSbom, error) {
+func (b *EntBackend) IngestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, spec model.HasSBOMInputSpec, includes model.HasSBOMIncludesInputSpec) (*model.HasSbom, error) {
+ // TODO(knrc) - handle includes
  funcName := "IngestHasSbom"
 
  sbomId, err := WithinTX(ctx, b.client, func(ctx context.Context) (*int, error) {
@@ -158,7 +159,7 @@ func (b *EntBackend) IngestHasSbom(ctx context.Context, subject model.PackageOrA
  return toModelHasSBOM(sbom), nil
 }
 
-func (b *EntBackend) IngestHasSBOMs(ctx context.Context, subjects model.PackageOrArtifactInputs, hasSBOMs []*model.HasSBOMInputSpec) ([]*model.HasSbom, error) {
+func (b *EntBackend) IngestHasSBOMs(ctx context.Context, subjects model.PackageOrArtifactInputs, hasSBOMs []*model.HasSBOMInputSpec, includes []*model.HasSBOMIncludesInputSpec) ([]*model.HasSbom, error) {
  var modelHasSboms []*model.HasSbom
  for i, hasSbom := range hasSBOMs {
  var subject model.PackageOrArtifactInput
@@ -167,7 +168,8 @@ func (b *EntBackend) IngestHasSBOMs(ctx context.Context, subjects model.PackageO
  } else {
  subject = model.PackageOrArtifactInput{Package: subjects.Packages[i]}
  }
- modelHasSbom, err := b.IngestHasSbom(ctx, subject, *hasSbom)
+ // TODO(knrc) - handle includes
+ modelHasSbom, err := b.IngestHasSbom(ctx, subject, *hasSbom, model.HasSBOMIncludesInputSpec{})
  if err != nil {
  return nil, gqlerror.Errorf("IngestHasSBOMs failed with err: %v", err)
  }

diff --git a/pkg/assembler/backends/ent/backend/sbom_test.go b/pkg/assembler/backends/ent/backend/sbom_test.go
@@ -30,6 +30,7 @@ func (s *Suite) Test_HasSBOM() {
  type call struct {
  Sub model.PackageOrArtifactInput
  Spec *model.HasSBOMInputSpec
+ Inc *model.HasSBOMIncludesInputSpec
  }
  tests := []struct {
  Name string
@@ -505,7 +506,8 @@ func (s *Suite) Test_HasSBOM() {
 
  recordIDs := make([]string, len(test.Calls))
  for i, o := range test.Calls {
- dep, err := b.IngestHasSbom(ctx, o.Sub, *o.Spec)
+ // TODO (knrc) handle includes
+ dep, err := b.IngestHasSbom(ctx, o.Sub, *o.Spec, model.HasSBOMIncludesInputSpec{})
  if (err != nil) != test.ExpIngestErr {
  s.T().Fatalf("did not get expected ingest error, want: %v, got: %v", test.ExpIngestErr, err)
  }
@@ -545,6 +547,7 @@ func (s *Suite) TestIngestHasSBOMs() {
  type call struct {
  Sub model.PackageOrArtifactInputs
  HS []*model.HasSBOMInputSpec
+ Inc []*model.HasSBOMIncludesInputSpec
  }
  tests := []struct {
  Name string
@@ -768,7 +771,7 @@ func (s *Suite) TestIngestHasSBOMs() {
  }
  }
  for _, o := range test.Calls {
- _, err := b.IngestHasSBOMs(ctx, o.Sub, o.HS)
+ _, err := b.IngestHasSBOMs(ctx, o.Sub, o.HS, o.Inc)
  if (err != nil) != test.ExpIngestErr {
  t.Fatalf("did not get expected ingest error, want: %v, got: %v", test.ExpIngestErr, err)
  }

diff --git a/pkg/assembler/backends/inmem/backend.go b/pkg/assembler/backends/inmem/backend.go
@@ -20,6 +20,9 @@ import (
  "errors"
  "fmt"
  "math"
+ "reflect"
+ "slices"
+ "strconv"
  "strings"
  "sync"
  "sync/atomic"
@@ -201,3 +204,62 @@ func unlock(m *sync.RWMutex, readOnly bool) {
  m.Unlock()
  }
 }
+
+func parseIDs(ids []string) ([]uint32, error) {
+ keys := make([]uint32, 0, len(ids))
+ for _, id := range ids {
+ if key, err := parseID(id); err != nil {
+ return nil, err
+ } else {
+ keys = append(keys, key)
+ }
+ }
+ return keys, nil
+}
+
+func parseID(id string) (uint32, error) {
+ id64, err := strconv.ParseUint(id, 10, 32)
+ return uint32(id64), err
+}
+
+func sortAndRemoveDups(ids []uint32) []uint32 {
+ numIDs := len(ids)
+ if numIDs > 1 {
+ slices.Sort(ids)
+ nextIndex := 1
+ for index := 1; index < numIDs; index++ {
+ currentVal := ids[index]
+ if ids[index-1] != currentVal {
+ ids[nextIndex] = currentVal
+ nextIndex++
+ }
+ }
+ ids = ids[:nextIndex]
+ }
+ return ids
+}
+
+func (c *demoClient) getPackageVersionAndArtifacts(pkgOrArt []uint32) (pkgs []uint32, arts []uint32, err error) {
+ for _, id := range pkgOrArt {
+ switch entry := c.index[id].(type) {
+ case *pkgVersionNode:
+ pkgs = append(pkgs, entry.id)
+ case *artStruct:
+ arts = append(arts, entry.id)
+ default:
+ return nil, nil, fmt.Errorf("unexpected type in package or artifact list: %s", reflect.TypeOf(entry))
+ }
+ }
+
+ return
+}
+
+// IDs should be sorted
+func (c *demoClient) isIDPresent(id string, linkIDs []uint32) bool {
+ linkID, err := strconv.ParseUint(id, 10, 32)
+ if err != nil {
+ return false
+ }
+ _, found := slices.BinarySearch[[]uint32](linkIDs, uint32(linkID))
+ return found
+}