From c0614ec47ce2fc3b1f9559ccd3b3c9ba288a48ce Mon Sep 17 00:00:00 2001 From: Parth Patel <88045217+pxp928@users.noreply.github.com> Date: Thu, 3 Aug 2023 11:07:51 -0400 Subject: [PATCH] add `certifyBad` query, `certifyGood` ingestion and query, update bulk assembler (#1123) * add query for certifyBad Signed-off-by: pxp928 * remove root src and pkg and seperate query for subjects Signed-off-by: pxp928 * add certifyGood to arango Signed-off-by: pxp928 * update batch assembler to use certifyBad and certifyGood bulk ingestion Signed-off-by: pxp928 --------- Signed-off-by: pxp928 --- cmd/guacgql/cmd/ingest.go | 162 +++ pkg/assembler/backends/arangodb/artifact.go | 8 +- pkg/assembler/backends/arangodb/backend.go | 144 +-- pkg/assembler/backends/arangodb/certifyBad.go | 262 ++++- .../backends/arangodb/certifyGood.go | 1032 ++++++++++++++++- .../backends/arangodb/certifyScorecard.go | 4 + pkg/assembler/backends/arangodb/cve.go | 4 + pkg/assembler/backends/arangodb/ghsa.go | 4 + pkg/assembler/backends/arangodb/hasSBOM.go | 34 +- pkg/assembler/backends/arangodb/hasSLSA.go | 8 +- pkg/assembler/backends/arangodb/hashEqual.go | 8 +- .../backends/arangodb/isDependency.go | 34 +- .../backends/arangodb/isOccurrence.go | 54 +- pkg/assembler/backends/arangodb/osv.go | 4 + pkg/assembler/backends/arangodb/pkg.go | 77 +- pkg/assembler/backends/arangodb/src.go | 43 +- pkg/assembler/clients/helpers/bulk.go | 126 +- 17 files changed, 1741 insertions(+), 267 deletions(-) diff --git a/cmd/guacgql/cmd/ingest.go b/cmd/guacgql/cmd/ingest.go index f80c033705..27fd00ae8e 100644 --- a/cmd/guacgql/cmd/ingest.go +++ b/cmd/guacgql/cmd/ingest.go @@ -51,6 +51,7 @@ func ingestData(port int) { ingestCertifyBad(ctx, gqlclient) ingestCertifyBads(ctx, gqlclient) ingestCertifyGood(ctx, gqlclient) + ingestCertifyGoods(ctx, gqlclient) ingestHashEqual(ctx, gqlclient) ingestHasSBOM(ctx, gqlclient) ingestHasSourceAt(ctx, gqlclient) @@ -1220,6 +1221,167 @@ func ingestCertifyGood(ctx context.Context, client graphql.Client) { } } +func ingestCertifyGoods(ctx context.Context, client graphql.Client) { + logger := logging.FromContext(ctx) + opensslNs := "openssl.org" + opensslVersion := "3.0.3" + djangoNameSpace := "" + sourceTag := "v0.0.1" + ingestCertifyGood := []struct { + name string + pkg []model.PkgInputSpec + pkgMatchType model.MatchFlags + source []model.SourceInputSpec + artifact []model.ArtifactInputSpec + certifyGood []model.CertifyGoodInputSpec + }{ + { + name: "this package as this specific version has been audited", + pkg: []model.PkgInputSpec{ + { + Type: "conan", + Namespace: &opensslNs, + Name: "openssl", + Version: &opensslVersion, + Qualifiers: []model.PackageQualifierInputSpec{{Key: "user", Value: "bincrafters"}, {Key: "channel", Value: "stable"}}, + }, + { + Type: "conan", + Namespace: &opensslNs, + Name: "openssl", + Version: &opensslVersion, + Qualifiers: []model.PackageQualifierInputSpec{{Key: "user", Value: "bincrafters"}, {Key: "channel", Value: "stable"}}, + }, + }, + pkgMatchType: model.MatchFlags{ + Pkg: model.PkgMatchTypeSpecificVersion, + }, + certifyGood: []model.CertifyGoodInputSpec{ + { + Justification: "this package as this specific version has been audited", + Origin: "Demo ingestion", + Collector: "Demo ingestion", + }, + { + Justification: "this package as this specific version has been audited", + Origin: "Demo ingestion", + Collector: "Demo ingestion", + }, + }, + }, + { + name: "this package (all versions) is trusted", + pkg: []model.PkgInputSpec{ + { + Type: "pypi", + Namespace: &djangoNameSpace, + Name: "django", + }, + { + Type: "pypi", + Namespace: &djangoNameSpace, + Name: "django", + }, + }, + pkgMatchType: model.MatchFlags{ + Pkg: model.PkgMatchTypeAllVersions, + }, + certifyGood: []model.CertifyGoodInputSpec{ + { + Justification: "this package (all versions) is trusted", + Origin: "Demo ingestion", + Collector: "Demo ingestion", + }, + { + Justification: "this package (all versions) is trusted", + Origin: "Demo ingestion", + Collector: "Demo ingestion", + }, + }, + }, + { + name: "this source repo is trusted", + source: []model.SourceInputSpec{ + { + Type: "git", + Namespace: "github", + Name: "github.com/guacsec/guac", + Tag: &sourceTag, + }, + { + Type: "git", + Namespace: "github", + Name: "github.com/guacsec/guac", + Tag: &sourceTag, + }, + }, + certifyGood: []model.CertifyGoodInputSpec{ + { + Justification: "this source repo is trusted", + Origin: "Demo ingestion", + Collector: "Demo ingestion", + }, + { + Justification: "this source repo is trusted", + Origin: "Demo ingestion", + Collector: "Demo ingestion", + }, + }, + }, + { + name: "artifact is associated with an audited build", + artifact: []model.ArtifactInputSpec{ + { + Digest: "6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf", + Algorithm: "sha256", + }, + { + Digest: "6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf", + Algorithm: "sha256", + }, + }, + certifyGood: []model.CertifyGoodInputSpec{ + { + Justification: "this artifact is associated with an audited build", + Origin: "Demo ingestion", + Collector: "Demo ingestion", + }, + { + Justification: "this artifact is associated with an audited build", + Origin: "Demo ingestion", + Collector: "Demo ingestion", + }, + }, + }, + } + for _, ingest := range ingestCertifyGood { + if ingest.pkg != nil { + if _, err := model.IngestPackages(ctx, client, ingest.pkg); err != nil { + logger.Errorf("Error in ingesting package: %v\n", err) + } + if _, err := model.CertifyGoodPkgs(ctx, client, ingest.pkg, ingest.pkgMatchType, ingest.certifyGood); err != nil { + logger.Errorf("Error in ingesting: %v\n", err) + } + } else if ingest.source != nil { + if _, err := model.IngestSources(ctx, client, ingest.source); err != nil { + logger.Errorf("Error in ingesting source: %v\n", err) + } + if _, err := model.CertifyGoodSrcs(ctx, client, ingest.source, ingest.certifyGood); err != nil { + logger.Errorf("Error in ingesting: %v\n", err) + } + } else if ingest.artifact != nil { + if _, err := model.IngestArtifacts(ctx, client, ingest.artifact); err != nil { + logger.Errorf("Error in ingesting artifact: %v\n", err) + } + if _, err := model.CertifyGoodArtifacts(ctx, client, ingest.artifact, ingest.certifyGood); err != nil { + logger.Errorf("Error in ingesting: %v\n", err) + } + } else { + fmt.Printf("input missing for cve, osv or ghsa") + } + } +} + func ingestHashEqual(ctx context.Context, client graphql.Client) { logger := logging.FromContext(ctx) ingestHashEqual := []struct { diff --git a/pkg/assembler/backends/arangodb/artifact.go b/pkg/assembler/backends/arangodb/artifact.go index 00a6fa4d77..5ab1e80b7b 100644 --- a/pkg/assembler/backends/arangodb/artifact.go +++ b/pkg/assembler/backends/arangodb/artifact.go @@ -28,7 +28,7 @@ import ( func (c *arangoClient) Artifacts(ctx context.Context, artifactSpec *model.ArtifactSpec) ([]*model.Artifact, error) { values := map[string]any{} - arangoQueryBuilder := setArtifactMatchValues(nil, artifactSpec, values) + arangoQueryBuilder := setArtifactMatchValues(artifactSpec, values) arangoQueryBuilder.query.WriteString("\n") arangoQueryBuilder.query.WriteString(`RETURN { "id": art._id, @@ -46,10 +46,8 @@ func (c *arangoClient) Artifacts(ctx context.Context, artifactSpec *model.Artifa return getArtifacts(ctx, cursor) } -func setArtifactMatchValues(arangoQueryBuilder *arangoQueryBuilder, artifactSpec *model.ArtifactSpec, queryValues map[string]any) *arangoQueryBuilder { - if arangoQueryBuilder == nil { - arangoQueryBuilder = newForQuery(artifactsStr, "art") - } +func setArtifactMatchValues(artifactSpec *model.ArtifactSpec, queryValues map[string]any) *arangoQueryBuilder { + arangoQueryBuilder := newForQuery(artifactsStr, "art") if artifactSpec != nil { if artifactSpec.ID != nil { arangoQueryBuilder.filter("art", "_id", "==", "@id") diff --git a/pkg/assembler/backends/arangodb/backend.go b/pkg/assembler/backends/arangodb/backend.go index 9c03d7dc48..9e34cd6b03 100644 --- a/pkg/assembler/backends/arangodb/backend.go +++ b/pkg/assembler/backends/arangodb/backend.go @@ -42,8 +42,6 @@ const ( guacEmpty string = "guac-empty-@@" // Package collections - pkgHasTypeStr string = "pkgHasType" - pkgRootsStr string = "pkgRoots" pkgTypesStr string = "pkgTypes" pkgHasNamespaceStr string = "pkgHasNamespace" pkgNamespacesStr string = "pkgNamespaces" @@ -53,8 +51,6 @@ const ( pkgVersionsStr string = "pkgVersions" // source collections - srcHasTypeStr string = "srcHasType" - srcRootsStr string = "srcRoots" srcTypesStr string = "srcTypes" srcHasNamespaceStr string = "srcHasNamespace" srcNamespacesStr string = "srcNamespaces" @@ -151,24 +147,10 @@ type arangoQueryBuilder struct { query strings.Builder } -type pkgRootData struct { - Key string `json:"_key"` - Id string `json:"_id"` - PkgRoot string `json:"root"` -} - -type srcRootData struct { - Key string `json:"_key"` - Id string `json:"_id"` - SrcRoot string `json:"root"` -} - type arangoClient struct { - client driver.Client - db driver.Database - graph driver.Graph - pkgRoot *pkgRootData - srcRoot *srcRootData + client driver.Client + db driver.Database + graph driver.Graph } func arangoDBConnect(address, user, password string) (driver.Client, error) { @@ -234,11 +216,6 @@ func GetBackend(ctx context.Context, args backends.BackendArgs) (backends.Backen } else { // setup package collections - var pkgHasType driver.EdgeDefinition - pkgHasType.Collection = pkgHasTypeStr - pkgHasType.From = []string{pkgRootsStr} - pkgHasType.To = []string{pkgTypesStr} - var pkgHasNamespace driver.EdgeDefinition pkgHasNamespace.Collection = pkgHasNamespaceStr pkgHasNamespace.From = []string{pkgTypesStr} @@ -255,11 +232,6 @@ func GetBackend(ctx context.Context, args backends.BackendArgs) (backends.Backen pkgHasVersion.To = []string{pkgVersionsStr} // setup source collections - var srcHasType driver.EdgeDefinition - srcHasType.Collection = srcHasTypeStr - srcHasType.From = []string{srcRootsStr} - srcHasType.To = []string{srcTypesStr} - var srcHasNamespace driver.EdgeDefinition srcHasNamespace.Collection = srcHasNamespaceStr srcHasNamespace.From = []string{srcTypesStr} @@ -376,23 +348,23 @@ func GetBackend(ctx context.Context, args backends.BackendArgs) (backends.Backen var certifyGoodPkgNameEdges driver.EdgeDefinition certifyGoodPkgNameEdges.Collection = certifyGoodPkgNameEdgesStr - certifyGoodPkgNameEdges.From = []string{pkgNamesStr, pkgVersionsStr, artifactsStr, srcNamesStr} + certifyGoodPkgNameEdges.From = []string{pkgNamesStr} certifyGoodPkgNameEdges.To = []string{certifyGoodsStr} var certifyGoodArtEdges driver.EdgeDefinition certifyGoodArtEdges.Collection = certifyGoodArtEdgesStr - certifyGoodArtEdges.From = []string{pkgNamesStr, pkgVersionsStr, artifactsStr, srcNamesStr} + certifyGoodArtEdges.From = []string{artifactsStr} certifyGoodArtEdges.To = []string{certifyGoodsStr} var certifyGoodSrcEdges driver.EdgeDefinition certifyGoodSrcEdges.Collection = certifyGoodSrcEdgesStr - certifyGoodSrcEdges.From = []string{pkgNamesStr, pkgVersionsStr, artifactsStr, srcNamesStr} + certifyGoodSrcEdges.From = []string{srcNamesStr} certifyGoodSrcEdges.To = []string{certifyGoodsStr} // A graph can contain additional vertex collections, defined in the set of orphan collections var options driver.CreateGraphOptions - options.EdgeDefinitions = []driver.EdgeDefinition{pkgHasType, pkgHasNamespace, pkgHasName, - pkgHasVersion, srcHasType, srcHasNamespace, srcHasName, isDependencyDepPkgEdges, isDependencySubjectPkgEdges, + options.EdgeDefinitions = []driver.EdgeDefinition{pkgHasNamespace, pkgHasName, + pkgHasVersion, srcHasNamespace, srcHasName, isDependencyDepPkgEdges, isDependencySubjectPkgEdges, isOccurrenceArtEdges, isOccurrenceSubjectPkgEdges, isOccurrenceSubjectSrcEdges, hasSLSASubjectArtEdges, hasSLSABuiltByEdges, hasSLSABuiltFromEdges, hashEqualArtEdges, hashEqualSubjectArtEdges, hasSBOMPkgEdges, hasSBOMArtEdges, certifyVulnEdges, certifyScorecardSrcEdges, certifyBadPkgVersionEdges, certifyBadPkgNameEdges, @@ -435,10 +407,6 @@ func GetBackend(ctx context.Context, args backends.BackendArgs) (backends.Backen return nil, fmt.Errorf("failed to generate index for hashEquals: %w", err) } - if err := createIndexPerCollection(ctx, db, pkgTypesStr, []string{"_parent", "type"}, true, "byPkgTypeParent"); err != nil { - return nil, fmt.Errorf("failed to generate index for pkgTypes: %w", err) - } - if err := createIndexPerCollection(ctx, db, pkgTypesStr, []string{"type"}, true, "byPkgType"); err != nil { return nil, fmt.Errorf("failed to generate index for pkgTypes: %w", err) } @@ -463,10 +431,6 @@ func GetBackend(ctx context.Context, args backends.BackendArgs) (backends.Backen return nil, fmt.Errorf("failed to generate index for pkgVersions: %w", err) } - if err := createIndexPerCollection(ctx, db, srcTypesStr, []string{"_parent", "type"}, true, "bySrcTypeParent"); err != nil { - return nil, fmt.Errorf("failed to generate index for srcTypes: %w", err) - } - if err := createIndexPerCollection(ctx, db, srcTypesStr, []string{"type"}, true, "bySrcType"); err != nil { return nil, fmt.Errorf("failed to generate index for srcTypes: %w", err) } @@ -583,17 +547,7 @@ func GetBackend(ctx context.Context, args backends.BackendArgs) (backends.Backen } } - collectedPkgRootData, err := preIngestPkgRoot(ctx, db) - if err != nil { - return nil, fmt.Errorf("failed to create package root: %w", err) - } - - collectedSrcRootData, err := preIngestSrcRoot(ctx, db) - if err != nil { - return nil, fmt.Errorf("failed to create source root: %w", err) - } - - arangoClient := &arangoClient{client: arangodbClient, db: db, graph: graph, pkgRoot: collectedPkgRootData, srcRoot: collectedSrcRootData} + arangoClient := &arangoClient{client: arangodbClient, db: db, graph: graph} return arangoClient, nil } @@ -672,14 +626,6 @@ func (aqb *arangoQueryBuilder) forInBound(edgeCollectionName string, counterVert return aqb } -func (aqb *arangoQueryBuilder) forInBoundWithEdgeCounter(edgeCollectionName string, counterVertexName string, counterEdgeName string, inBoundStartVertexName string) *arangoQueryBuilder { - aqb.query.WriteString("\n") - - aqb.query.WriteString(fmt.Sprintf("FOR %s, %s IN INBOUND %s %s", counterVertexName, counterEdgeName, inBoundStartVertexName, edgeCollectionName)) - - return aqb -} - func (aqb *arangoQueryBuilder) filter(counterName string, fieldName string, condition string, value string) *arangoQueryFilter { aqb.query.WriteString(" ") @@ -815,78 +761,6 @@ func (c *arangoClient) Path(ctx context.Context, subject string, target string, panic(fmt.Errorf("not implemented: Path - Path")) } -func preIngestPkgRoot(ctx context.Context, db driver.Database) (*pkgRootData, error) { - query := ` - UPSERT { root: "pkg" } - INSERT { root: "pkg" } - UPDATE {} - IN pkgRoots - RETURN NEW` - - cursor, err := executeQueryWithRetry(ctx, db, query, nil, "preIngestPkgRoot") - if err != nil { - return nil, fmt.Errorf("failed to ingest package root node: %w", err) - } - - var createdValues []pkgRootData - for { - var doc pkgRootData - _, err := cursor.ReadDocument(ctx, &doc) - if err != nil { - if driver.IsNoMoreDocuments(err) { - cursor.Close() - break - } else { - return nil, fmt.Errorf("failed to ingest pkg root: %w", err) - } - } else { - createdValues = append(createdValues, doc) - } - } - - if len(createdValues) == 1 { - return &createdValues[0], nil - } else { - return nil, fmt.Errorf("number of pkg root ingested is greater than one") - } -} - -func preIngestSrcRoot(ctx context.Context, db driver.Database) (*srcRootData, error) { - query := ` - UPSERT { root: "src" } - INSERT { root: "src" } - UPDATE {} - IN srcRoots - RETURN NEW` - - cursor, err := executeQueryWithRetry(ctx, db, query, nil, "preIngestSrcRoot") - if err != nil { - return nil, fmt.Errorf("failed to ingest source root node: %w", err) - } - - var createdValues []srcRootData - for { - var doc srcRootData - _, err := cursor.ReadDocument(ctx, &doc) - if err != nil { - if driver.IsNoMoreDocuments(err) { - cursor.Close() - break - } else { - return nil, fmt.Errorf("failed to ingest src root: %w", err) - } - } else { - createdValues = append(createdValues, doc) - } - } - - if len(createdValues) == 1 { - return &createdValues[0], nil - } else { - return nil, fmt.Errorf("number of src root ingested is greater than one") - } -} - func ptrfromArangoSearchNGramStreamType(s driver.ArangoSearchNGramStreamType) *driver.ArangoSearchNGramStreamType { return &s } diff --git a/pkg/assembler/backends/arangodb/certifyBad.go b/pkg/assembler/backends/arangodb/certifyBad.go index 2032612140..d3ad7f5124 100644 --- a/pkg/assembler/backends/arangodb/certifyBad.go +++ b/pkg/assembler/backends/arangodb/certifyBad.go @@ -26,23 +26,255 @@ import ( ) func (c *arangoClient) CertifyBad(ctx context.Context, certifyBadSpec *model.CertifyBadSpec) ([]*model.CertifyBad, error) { - return []*model.CertifyBad{}, fmt.Errorf("not implemented - CertifyBad") + + var arangoQueryBuilder *arangoQueryBuilder + if certifyBadSpec.Subject != nil { + var combinedCertifyBad []*model.CertifyBad + if certifyBadSpec.Subject.Package != nil { + values := map[string]any{} + // pkgVersion certifyBad + arangoQueryBuilder = setPkgVersionMatchValues(certifyBadSpec.Subject.Package, values) + arangoQueryBuilder.forOutBound(certifyBadPkgVersionEdgesStr, "certifyBad", "pVersion") + setCertifyBadMatchValues(arangoQueryBuilder, certifyBadSpec, values) + + pkgVersionCertifyBads, err := getPkgVersionCertifyBadForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package version certifyBad with error: %w", err) + } + + combinedCertifyBad = append(combinedCertifyBad, pkgVersionCertifyBads...) + + // pkgName certifyBad + arangoQueryBuilder = setPkgNameMatchValues(certifyBadSpec.Subject.Package, values) + arangoQueryBuilder.forOutBound(certifyBadPkgNameEdgesStr, "certifyBad", "pName") + setCertifyBadMatchValues(arangoQueryBuilder, certifyBadSpec, values) + + pkgNameCertifyBads, err := getPkgNameCertifyBadForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package name certifyBad with error: %w", err) + } + + combinedCertifyBad = append(combinedCertifyBad, pkgNameCertifyBads...) + } + if certifyBadSpec.Subject.Source != nil { + values := map[string]any{} + arangoQueryBuilder = setSrcMatchValues(certifyBadSpec.Subject.Source, values) + arangoQueryBuilder.forOutBound(certifyBadSrcEdgesStr, "certifyBad", "sName") + setCertifyBadMatchValues(arangoQueryBuilder, certifyBadSpec, values) + + srcCertifyBads, err := getSrcCertifyBadForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve source certifyBad with error: %w", err) + } + + combinedCertifyBad = append(combinedCertifyBad, srcCertifyBads...) + } + if certifyBadSpec.Subject.Artifact != nil { + values := map[string]any{} + arangoQueryBuilder = setArtifactMatchValues(certifyBadSpec.Subject.Artifact, values) + arangoQueryBuilder.forOutBound(certifyBadArtEdgesStr, "certifyBad", "art") + setCertifyBadMatchValues(arangoQueryBuilder, certifyBadSpec, values) + + artCertifyBads, err := getArtCertifyBadForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve artifact certifyBad with error: %w", err) + } + + combinedCertifyBad = append(combinedCertifyBad, artCertifyBads...) + } + return combinedCertifyBad, nil + } else { + values := map[string]any{} + var combinedCertifyBad []*model.CertifyBad + + // pkgVersion certifyBad + arangoQueryBuilder = newForQuery(certifyBadsStr, "certifyBad") + setCertifyBadMatchValues(arangoQueryBuilder, certifyBadSpec, values) + arangoQueryBuilder.forInBound(certifyBadPkgVersionEdgesStr, "pVersion", "certifyBad") + arangoQueryBuilder.forInBound(pkgHasVersionStr, "pName", "pVersion") + arangoQueryBuilder.forInBound(pkgHasNameStr, "pNs", "pName") + arangoQueryBuilder.forInBound(pkgHasNamespaceStr, "pType", "pNs") + + pkgVersionCertifyBads, err := getPkgVersionCertifyBadForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package version certifyBad with error: %w", err) + } + combinedCertifyBad = append(combinedCertifyBad, pkgVersionCertifyBads...) + + // pkgName certifyBad + arangoQueryBuilder = newForQuery(certifyBadsStr, "certifyBad") + setCertifyBadMatchValues(arangoQueryBuilder, certifyBadSpec, values) + arangoQueryBuilder.forInBound(certifyBadPkgNameEdgesStr, "pName", "certifyBad") + arangoQueryBuilder.forInBound(pkgHasNameStr, "pNs", "pName") + arangoQueryBuilder.forInBound(pkgHasNamespaceStr, "pType", "pNs") + + pkgNameCertifyBads, err := getPkgNameCertifyBadForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package name certifyBad with error: %w", err) + } + combinedCertifyBad = append(combinedCertifyBad, pkgNameCertifyBads...) + + // get sources + arangoQueryBuilder = newForQuery(certifyBadsStr, "certifyBad") + setCertifyBadMatchValues(arangoQueryBuilder, certifyBadSpec, values) + arangoQueryBuilder.forInBound(certifyBadSrcEdgesStr, "sName", "certifyBad") + arangoQueryBuilder.forInBound(srcHasNameStr, "sNs", "sName") + arangoQueryBuilder.forInBound(srcHasNamespaceStr, "sType", "sNs") + + srcCertifyBads, err := getSrcCertifyBadForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve source certifyBad with error: %w", err) + } + combinedCertifyBad = append(combinedCertifyBad, srcCertifyBads...) + + // get artifacts + arangoQueryBuilder = newForQuery(certifyBadsStr, "certifyBad") + setCertifyBadMatchValues(arangoQueryBuilder, certifyBadSpec, values) + arangoQueryBuilder.forInBound(certifyBadArtEdgesStr, "art", "certifyBad") + + artCertifyBads, err := getArtCertifyBadForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve artifact certifyBad with error: %w", err) + } + combinedCertifyBad = append(combinedCertifyBad, artCertifyBads...) + + return combinedCertifyBad, nil + } } -// func setCertifyBadMatchValues(arangoQueryBuilder *arangoQueryBuilder, certifyBadSpec *model.CertifyBadSpec, queryValues map[string]any) { -// if certifyBadSpec.Justification != nil { -// arangoQueryBuilder.filter("bad", justification, "==", "@"+justification) -// queryValues[justification] = certifyBadSpec.Justification -// } -// if certifyBadSpec.Origin != nil { -// arangoQueryBuilder.filter("bad", origin, "==", "@"+origin) -// queryValues[origin] = certifyBadSpec.Origin -// } -// if certifyBadSpec.Collector != nil { -// arangoQueryBuilder.filter("bad", collector, "==", "@"+collector) -// queryValues[collector] = certifyBadSpec.Collector -// } -// } +func getSrcCertifyBadForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.CertifyBad, error) { + arangoQueryBuilder.query.WriteString("\n") + arangoQueryBuilder.query.WriteString(`RETURN { + 'srcName': { + 'type_id': sType._id, + 'type': sType.type, + 'namespace_id': sNs._id, + 'namespace': sNs.namespace, + 'name_id': sName._id, + 'name': sName.name, + 'commit': sName.commit, + 'tag': sName.tag + }, + 'certifyBad_id': certifyBad._id, + 'justification': certifyBad.justification, + 'collector': certifyBad.collector, + 'origin': certifyBad.origin + }`) + + fmt.Println(arangoQueryBuilder.string()) + + cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "CertifyBad") + if err != nil { + return nil, fmt.Errorf("failed to query for CertifyBad: %w", err) + } + defer cursor.Close() + + return getSourceCertifyBad(ctx, cursor) +} + +func getArtCertifyBadForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.CertifyBad, error) { + arangoQueryBuilder.query.WriteString("\n") + arangoQueryBuilder.query.WriteString(`RETURN { + 'artifact': { + 'id': art._id, + 'algorithm': art.algorithm, + 'digest': art.digest + }, + 'certifyBad_id': certifyBad._id, + 'justification': certifyBad.justification, + 'collector': certifyBad.collector, + 'origin': certifyBad.origin + }`) + + fmt.Println(arangoQueryBuilder.string()) + + cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "CertifyBad") + if err != nil { + return nil, fmt.Errorf("failed to query for CertifyBad: %w", err) + } + defer cursor.Close() + + return getArtifactCertifyBad(ctx, cursor) +} + +func getPkgNameCertifyBadForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.CertifyBad, error) { + arangoQueryBuilder.query.WriteString("\n") + arangoQueryBuilder.query.WriteString(`RETURN { + 'pkgName': { + 'type_id': pType._id, + 'type': pType.type, + 'namespace_id': pNs._id, + 'namespace': pNs.namespace, + 'name_id': pName._id, + 'name': pName.name + }, + 'certifyBad_id': certifyBad._id, + 'justification': certifyBad.justification, + 'collector': certifyBad.collector, + 'origin': certifyBad.origin + }`) + + fmt.Println(arangoQueryBuilder.string()) + + cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "CertifyBad") + if err != nil { + return nil, fmt.Errorf("failed to query for CertifyBad: %w", err) + } + defer cursor.Close() + + return getPkgNameCertifyBad(ctx, cursor) +} + +func getPkgVersionCertifyBadForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.CertifyBad, error) { + arangoQueryBuilder.query.WriteString("\n") + arangoQueryBuilder.query.WriteString(`RETURN { + 'pkgVersion': { + 'type_id': pType._id, + 'type': pType.type, + 'namespace_id': pNs._id, + 'namespace': pNs.namespace, + 'name_id': pName._id, + 'name': pName.name, + 'version_id': pVersion._id, + 'version': pVersion.version, + 'subpath': pVersion.subpath, + 'qualifier_list': pVersion.qualifier_list + }, + 'certifyBad_id': certifyBad._id, + 'justification': certifyBad.justification, + 'collector': certifyBad.collector, + 'origin': certifyBad.origin + }`) + + fmt.Println(arangoQueryBuilder.string()) + + cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "CertifyBad") + if err != nil { + return nil, fmt.Errorf("failed to query for CertifyBad: %w", err) + } + defer cursor.Close() + + return getPkgVersionCertifyBad(ctx, cursor) +} + +func setCertifyBadMatchValues(arangoQueryBuilder *arangoQueryBuilder, certifyBadSpec *model.CertifyBadSpec, queryValues map[string]any) { + if certifyBadSpec.ID != nil { + arangoQueryBuilder.filter("certifyBad", "_id", "==", "@id") + queryValues["id"] = *certifyBadSpec.ID + } + if certifyBadSpec.Justification != nil { + arangoQueryBuilder.filter("certifyBad", justification, "==", "@"+justification) + queryValues[justification] = certifyBadSpec.Justification + } + if certifyBadSpec.Origin != nil { + arangoQueryBuilder.filter("certifyBad", origin, "==", "@"+origin) + queryValues[origin] = certifyBadSpec.Origin + } + if certifyBadSpec.Collector != nil { + arangoQueryBuilder.filter("certifyBad", collector, "==", "@"+collector) + queryValues[collector] = certifyBadSpec.Collector + } +} func getCertifyBadQueryValues(pkg *model.PkgInputSpec, pkgMatchType *model.MatchFlags, artifact *model.ArtifactInputSpec, source *model.SourceInputSpec, certifyBad *model.CertifyBadInputSpec) map[string]any { values := map[string]any{} diff --git a/pkg/assembler/backends/arangodb/certifyGood.go b/pkg/assembler/backends/arangodb/certifyGood.go index 78219c384c..97684c2c91 100644 --- a/pkg/assembler/backends/arangodb/certifyGood.go +++ b/pkg/assembler/backends/arangodb/certifyGood.go @@ -17,19 +17,1045 @@ package arangodb import ( "context" + "encoding/json" "fmt" + "strings" + "github.com/arangodb/go-driver" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) func (c *arangoClient) CertifyGood(ctx context.Context, certifyGoodSpec *model.CertifyGoodSpec) ([]*model.CertifyGood, error) { - panic(fmt.Errorf("not implemented: CertifyGood - CertifyGood")) + var arangoQueryBuilder *arangoQueryBuilder + if certifyGoodSpec.Subject != nil { + var combinedCertifyGood []*model.CertifyGood + if certifyGoodSpec.Subject.Package != nil { + values := map[string]any{} + // pkgVersion certifyGood + arangoQueryBuilder = setPkgVersionMatchValues(certifyGoodSpec.Subject.Package, values) + arangoQueryBuilder.forOutBound(certifyGoodPkgVersionEdgesStr, "certifyGood", "pVersion") + setCertifyGoodMatchValues(arangoQueryBuilder, certifyGoodSpec, values) + + pkgVersionCertifyGoods, err := getPkgVersionCertifyGoodForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package version certifyGood with error: %w", err) + } + + combinedCertifyGood = append(combinedCertifyGood, pkgVersionCertifyGoods...) + + // pkgName certifyGood + arangoQueryBuilder = setPkgNameMatchValues(certifyGoodSpec.Subject.Package, values) + arangoQueryBuilder.forOutBound(certifyGoodPkgNameEdgesStr, "certifyGood", "pName") + setCertifyGoodMatchValues(arangoQueryBuilder, certifyGoodSpec, values) + + pkgNameCertifyGoods, err := getPkgNameCertifyGoodForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package name certifyGood with error: %w", err) + } + + combinedCertifyGood = append(combinedCertifyGood, pkgNameCertifyGoods...) + } + if certifyGoodSpec.Subject.Source != nil { + values := map[string]any{} + arangoQueryBuilder = setSrcMatchValues(certifyGoodSpec.Subject.Source, values) + arangoQueryBuilder.forOutBound(certifyGoodSrcEdgesStr, "certifyGood", "sName") + setCertifyGoodMatchValues(arangoQueryBuilder, certifyGoodSpec, values) + + srcCertifyGoods, err := getSrcCertifyGoodForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve source certifyGood with error: %w", err) + } + + combinedCertifyGood = append(combinedCertifyGood, srcCertifyGoods...) + } + if certifyGoodSpec.Subject.Artifact != nil { + values := map[string]any{} + arangoQueryBuilder = setArtifactMatchValues(certifyGoodSpec.Subject.Artifact, values) + arangoQueryBuilder.forOutBound(certifyGoodArtEdgesStr, "certifyGood", "art") + setCertifyGoodMatchValues(arangoQueryBuilder, certifyGoodSpec, values) + + artCertifyGoods, err := getArtCertifyGoodForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve artifact certifyGood with error: %w", err) + } + + combinedCertifyGood = append(combinedCertifyGood, artCertifyGoods...) + } + return combinedCertifyGood, nil + } else { + values := map[string]any{} + var combinedCertifyGood []*model.CertifyGood + + // pkgVersion certifyGood + arangoQueryBuilder = newForQuery(certifyGoodsStr, "certifyGood") + setCertifyGoodMatchValues(arangoQueryBuilder, certifyGoodSpec, values) + arangoQueryBuilder.forInBound(certifyGoodPkgVersionEdgesStr, "pVersion", "certifyGood") + arangoQueryBuilder.forInBound(pkgHasVersionStr, "pName", "pVersion") + arangoQueryBuilder.forInBound(pkgHasNameStr, "pNs", "pName") + arangoQueryBuilder.forInBound(pkgHasNamespaceStr, "pType", "pNs") + + pkgVersionCertifyGoods, err := getPkgVersionCertifyGoodForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package version certifyGood with error: %w", err) + } + combinedCertifyGood = append(combinedCertifyGood, pkgVersionCertifyGoods...) + + // pkgName certifyGood + arangoQueryBuilder = newForQuery(certifyGoodsStr, "certifyGood") + setCertifyGoodMatchValues(arangoQueryBuilder, certifyGoodSpec, values) + arangoQueryBuilder.forInBound(certifyGoodPkgNameEdgesStr, "pName", "certifyGood") + arangoQueryBuilder.forInBound(pkgHasNameStr, "pNs", "pName") + arangoQueryBuilder.forInBound(pkgHasNamespaceStr, "pType", "pNs") + + pkgNameCertifyGoods, err := getPkgNameCertifyGoodForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package name certifyGood with error: %w", err) + } + combinedCertifyGood = append(combinedCertifyGood, pkgNameCertifyGoods...) + + // get sources + arangoQueryBuilder = newForQuery(certifyGoodsStr, "certifyGood") + setCertifyGoodMatchValues(arangoQueryBuilder, certifyGoodSpec, values) + arangoQueryBuilder.forInBound(certifyGoodSrcEdgesStr, "sName", "certifyGood") + arangoQueryBuilder.forInBound(srcHasNameStr, "sNs", "sName") + arangoQueryBuilder.forInBound(srcHasNamespaceStr, "sType", "sNs") + + srcCertifyGoods, err := getSrcCertifyGoodForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve source certifyGood with error: %w", err) + } + combinedCertifyGood = append(combinedCertifyGood, srcCertifyGoods...) + + // get artifacts + arangoQueryBuilder = newForQuery(certifyGoodsStr, "certifyGood") + setCertifyGoodMatchValues(arangoQueryBuilder, certifyGoodSpec, values) + arangoQueryBuilder.forInBound(certifyGoodArtEdgesStr, "art", "certifyGood") + + artCertifyGoods, err := getArtCertifyGoodForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve artifact certifyGood with error: %w", err) + } + combinedCertifyGood = append(combinedCertifyGood, artCertifyGoods...) + + return combinedCertifyGood, nil + } +} + +func getSrcCertifyGoodForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.CertifyGood, error) { + arangoQueryBuilder.query.WriteString("\n") + arangoQueryBuilder.query.WriteString(`RETURN { + 'srcName': { + 'type_id': sType._id, + 'type': sType.type, + 'namespace_id': sNs._id, + 'namespace': sNs.namespace, + 'name_id': sName._id, + 'name': sName.name, + 'commit': sName.commit, + 'tag': sName.tag + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }`) + + fmt.Println(arangoQueryBuilder.string()) + + cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "certifyGood") + if err != nil { + return nil, fmt.Errorf("failed to query for certifyGood: %w", err) + } + defer cursor.Close() + + return getSourceCertifyGood(ctx, cursor) +} + +func getArtCertifyGoodForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.CertifyGood, error) { + arangoQueryBuilder.query.WriteString("\n") + arangoQueryBuilder.query.WriteString(`RETURN { + 'artifact': { + 'id': art._id, + 'algorithm': art.algorithm, + 'digest': art.digest + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }`) + + fmt.Println(arangoQueryBuilder.string()) + + cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "certifyGood") + if err != nil { + return nil, fmt.Errorf("failed to query for certifyGood: %w", err) + } + defer cursor.Close() + + return getArtifactCertifyGood(ctx, cursor) +} + +func getPkgNameCertifyGoodForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.CertifyGood, error) { + arangoQueryBuilder.query.WriteString("\n") + arangoQueryBuilder.query.WriteString(`RETURN { + 'pkgName': { + 'type_id': pType._id, + 'type': pType.type, + 'namespace_id': pNs._id, + 'namespace': pNs.namespace, + 'name_id': pName._id, + 'name': pName.name + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }`) + + fmt.Println(arangoQueryBuilder.string()) + + cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "certifyGood") + if err != nil { + return nil, fmt.Errorf("failed to query for certifyGood: %w", err) + } + defer cursor.Close() + + return getPkgNameCertifyGood(ctx, cursor) +} + +func getPkgVersionCertifyGoodForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.CertifyGood, error) { + arangoQueryBuilder.query.WriteString("\n") + arangoQueryBuilder.query.WriteString(`RETURN { + 'pkgVersion': { + 'type_id': pType._id, + 'type': pType.type, + 'namespace_id': pNs._id, + 'namespace': pNs.namespace, + 'name_id': pName._id, + 'name': pName.name, + 'version_id': pVersion._id, + 'version': pVersion.version, + 'subpath': pVersion.subpath, + 'qualifier_list': pVersion.qualifier_list + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }`) + + fmt.Println(arangoQueryBuilder.string()) + + cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "CertifyGood") + if err != nil { + return nil, fmt.Errorf("failed to query for CertifyGood: %w", err) + } + defer cursor.Close() + + return getPkgVersionCertifyGood(ctx, cursor) +} + +func setCertifyGoodMatchValues(arangoQueryBuilder *arangoQueryBuilder, certifyGoodSpec *model.CertifyGoodSpec, queryValues map[string]any) { + if certifyGoodSpec.ID != nil { + arangoQueryBuilder.filter("certifyGood", "_id", "==", "@id") + queryValues["id"] = *certifyGoodSpec.ID + } + if certifyGoodSpec.Justification != nil { + arangoQueryBuilder.filter("certifyGood", justification, "==", "@"+justification) + queryValues[justification] = certifyGoodSpec.Justification + } + if certifyGoodSpec.Origin != nil { + arangoQueryBuilder.filter("certifyGood", origin, "==", "@"+origin) + queryValues[origin] = certifyGoodSpec.Origin + } + if certifyGoodSpec.Collector != nil { + arangoQueryBuilder.filter("certifyGood", collector, "==", "@"+collector) + queryValues[collector] = certifyGoodSpec.Collector + } +} + +func getCertifyGoodQueryValues(pkg *model.PkgInputSpec, pkgMatchType *model.MatchFlags, artifact *model.ArtifactInputSpec, source *model.SourceInputSpec, certifyGood *model.CertifyGoodInputSpec) map[string]any { + values := map[string]any{} + // add guac keys + if pkg != nil { + pkgId := guacPkgId(*pkg) + if pkgMatchType.Pkg == model.PkgMatchTypeAllVersions { + values["pkgNameGuacKey"] = pkgId.NameId + } else { + values["pkgVersionGuacKey"] = pkgId.VersionId + } + } else if artifact != nil { + values["art_algorithm"] = strings.ToLower(artifact.Algorithm) + values["art_digest"] = strings.ToLower(artifact.Digest) + } else { + source := guacSrcId(*source) + values["srcNameGuacKey"] = source.NameId + } + + values["justification"] = certifyGood.Justification + values["origin"] = certifyGood.Origin + values["collector"] = certifyGood.Collector + + return values } func (c *arangoClient) IngestCertifyGood(ctx context.Context, subject model.PackageSourceOrArtifactInput, pkgMatchType *model.MatchFlags, certifyGood model.CertifyGoodInputSpec) (*model.CertifyGood, error) { - panic(fmt.Errorf("not implemented: IngestCertifyGood - IngestCertifyGood")) + if subject.Package != nil { + if pkgMatchType.Pkg == model.PkgMatchTypeSpecificVersion { + query := ` + LET firstPkg = FIRST( + FOR pVersion in pkgVersions + FILTER pVersion.guacKey == @pkgVersionGuacKey + FOR pName in pkgNames + FILTER pName._id == pVersion._parent + FOR pNs in pkgNamespaces + FILTER pNs._id == pName._parent + FOR pType in pkgTypes + FILTER pType._id == pNs._parent + + RETURN { + 'typeID': pType._id, + 'type': pType.type, + 'namespace_id': pNs._id, + 'namespace': pNs.namespace, + 'name_id': pName._id, + 'name': pName.name, + 'version_id': pVersion._id, + 'version': pVersion.version, + 'subpath': pVersion.subpath, + 'qualifier_list': pVersion.qualifier_list, + 'versionDoc': pVersion + } + ) + + LET certifyGood = FIRST( + UPSERT { packageID:firstPkg.version_id, justification:@justification, collector:@collector, origin:@origin } + INSERT { packageID:firstPkg.version_id, justification:@justification, collector:@collector, origin:@origin } + UPDATE {} IN certifyGoods + RETURN NEW + ) + + LET edgeCollection = ( + INSERT { _key: CONCAT("certifyGoodPkgVersionEdges", firstPkg.versionDoc._key, certifyGood._key), _from: firstPkg.version_id, _to: certifyGood._id } INTO certifyGoodPkgVersionEdges OPTIONS { overwriteMode: "ignore" } + ) + + RETURN { + 'pkgVersion': { + 'type_id': firstPkg.typeID, + 'type': firstPkg.type, + 'namespace_id': firstPkg.namespace_id, + 'namespace': firstPkg.namespace, + 'name_id': firstPkg.name_id, + 'name': firstPkg.name, + 'version_id': firstPkg.version_id, + 'version': firstPkg.version, + 'subpath': firstPkg.subpath, + 'qualifier_list': firstPkg.qualifier_list + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }` + + cursor, err := executeQueryWithRetry(ctx, c.db, query, getCertifyGoodQueryValues(subject.Package, pkgMatchType, nil, nil, &certifyGood), "IngestCertifyGood - PkgVersion") + if err != nil { + return nil, fmt.Errorf("failed to ingest package certifyGood: %w", err) + } + defer cursor.Close() + + certifyGoodList, err := getPkgVersionCertifyGood(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("failed to get certifyGoods from arango cursor: %w", err) + } + + if len(certifyGoodList) == 1 { + return certifyGoodList[0], nil + } else { + return nil, fmt.Errorf("number of certifyGood ingested is greater than one") + } + } else { + query := ` + LET firstPkg = FIRST( + FOR pName in pkgNames + FILTER pName.guacKey == @pkgNameGuacKey + FOR pNs in pkgNamespaces + FILTER pNs._id == pName._parent + FOR pType in pkgTypes + FILTER pType._id == pNs._parent + + RETURN { + 'typeID': pType._id, + 'type': pType.type, + 'namespace_id': pNs._id, + 'namespace': pNs.namespace, + 'name_id': pName._id, + 'name': pName.name, + 'nameDoc': pName + } + ) + + LET certifyGood = FIRST( + UPSERT { packageID:firstPkg.name_id, justification:@justification, collector:@collector, origin:@origin } + INSERT { packageID:firstPkg.name_id, justification:@justification, collector:@collector, origin:@origin } + UPDATE {} IN certifyGoods + RETURN NEW + ) + + LET edgeCollection = ( + INSERT { _key: CONCAT("certifyGoodPkgNameEdges", firstPkg.nameDoc._key, certifyGood._key), _from: firstPkg.name_id, _to: certifyGood._id } INTO certifyGoodPkgNameEdges OPTIONS { overwriteMode: "ignore" } + ) + + RETURN { + 'pkgName': { + 'type_id': firstPkg.typeID, + 'type': firstPkg.type, + 'namespace_id': firstPkg.namespace_id, + 'namespace': firstPkg.namespace, + 'name_id': firstPkg.name_id, + 'name': firstPkg.name + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }` + + cursor, err := executeQueryWithRetry(ctx, c.db, query, getCertifyGoodQueryValues(subject.Package, pkgMatchType, nil, nil, &certifyGood), "IngestCertifyGood - PkgName") + if err != nil { + return nil, fmt.Errorf("failed to ingest package certifyGood: %w", err) + } + defer cursor.Close() + + certifyGoodList, err := getPkgNameCertifyGood(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("failed to get certifyGoods from arango cursor: %w", err) + } + + if len(certifyGoodList) == 1 { + return certifyGoodList[0], nil + } else { + return nil, fmt.Errorf("number of certifyGood ingested is greater than one") + } + } + + } else if subject.Artifact != nil { + query := `LET artifact = FIRST(FOR art IN artifacts FILTER art.algorithm == @art_algorithm FILTER art.digest == @art_digest RETURN art) + + LET certifyGood = FIRST( + UPSERT { artifactID:artifact._id, justification:@justification, collector:@collector, origin:@origin } + INSERT { artifactID:artifact._id, justification:@justification, collector:@collector, origin:@origin } + UPDATE {} IN certifyGoods + RETURN NEW + ) + + LET edgeCollection = ( + INSERT { _key: CONCAT("certifyGoodArtEdges", artifact._key, certifyGood._key), _from: artifact._id, _to: certifyGood._id } INTO certifyGoodArtEdges OPTIONS { overwriteMode: "ignore" } + ) + + RETURN { + 'artifact': { + 'id': artifact._id, + 'algorithm': artifact.algorithm, + 'digest': artifact.digest + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }` + + cursor, err := executeQueryWithRetry(ctx, c.db, query, getCertifyGoodQueryValues(nil, nil, subject.Artifact, nil, &certifyGood), "IngestCertifyGood - artifact") + if err != nil { + return nil, fmt.Errorf("failed to ingest artifact certifyGood: %w", err) + } + defer cursor.Close() + certifyGoodList, err := getArtifactCertifyGood(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("failed to get certifyGoods from arango cursor: %w", err) + } + + if len(certifyGoodList) == 1 { + return certifyGoodList[0], nil + } else { + return nil, fmt.Errorf("number of certifyGood ingested is greater than one") + } + + } else if subject.Source != nil { + query := ` + LET firstSrc = FIRST( + FOR sName in srcNames + FILTER sName.guacKey == @srcNameGuacKey + FOR sNs in srcNamespaces + FILTER sNs._id == sName._parent + FOR sType in srcTypes + FILTER sType._id == sNs._parent + + RETURN { + 'typeID': sType._id, + 'type': sType.type, + 'namespace_id': sNs._id, + 'namespace': sNs.namespace, + 'name_id': sName._id, + 'name': sName.name, + 'commit': sName.commit, + 'tag': sName.tag, + 'nameDoc': sName + } + ) + + LET certifyGood = FIRST( + UPSERT { sourceID:firstSrc.name_id, justification:@justification, collector:@collector, origin:@origin } + INSERT { sourceID:firstSrc.name_id, justification:@justification, collector:@collector, origin:@origin } + UPDATE {} IN certifyGoods + RETURN NEW + ) + + LET edgeCollection = ( + INSERT { _key: CONCAT("certifyGoodSrcEdges", firstSrc.nameDoc._key, certifyGood._key), _from: firstSrc.name_id, _to: certifyGood._id } INTO certifyGoodSrcEdges OPTIONS { overwriteMode: "ignore" } + ) + + RETURN { + 'srcName': { + 'type_id': firstSrc.typeID, + 'type': firstSrc.type, + 'namespace_id': firstSrc.namespace_id, + 'namespace': firstSrc.namespace, + 'name_id': firstSrc.name_id, + 'name': firstSrc.name, + 'commit': firstSrc.commit, + 'tag': firstSrc.tag + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }` + + cursor, err := executeQueryWithRetry(ctx, c.db, query, getCertifyGoodQueryValues(nil, nil, nil, subject.Source, &certifyGood), "IngestCertifyGood - source") + if err != nil { + return nil, fmt.Errorf("failed to ingest source certifyGood: %w", err) + } + defer cursor.Close() + certifyGoodList, err := getSourceCertifyGood(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("failed to get certifyGoods from arango cursor: %w", err) + } + + if len(certifyGoodList) == 1 { + return certifyGoodList[0], nil + } else { + return nil, fmt.Errorf("number of certifyGood ingested is greater than one") + } + + } else { + return nil, fmt.Errorf("package, artifact, or source is specified for IngestCertifyGood") + } } func (c *arangoClient) IngestCertifyGoods(ctx context.Context, subjects model.PackageSourceOrArtifactInputs, pkgMatchType *model.MatchFlags, certifyGoods []*model.CertifyGoodInputSpec) ([]*model.CertifyGood, error) { - panic(fmt.Errorf("not implemented: IngestCertifyGood - IngestCertifyGood")) + if len(subjects.Packages) > 0 { + if len(subjects.Packages) != len(certifyGoods) { + return nil, fmt.Errorf("uneven packages and certifyGoods for ingestion") + } + + var listOfValues []map[string]any + + for i := range subjects.Packages { + listOfValues = append(listOfValues, getCertifyGoodQueryValues(subjects.Packages[i], pkgMatchType, nil, nil, certifyGoods[i])) + } + + var documents []string + for _, val := range listOfValues { + bs, _ := json.Marshal(val) + documents = append(documents, string(bs)) + } + + queryValues := map[string]any{} + queryValues["documents"] = fmt.Sprint(strings.Join(documents, ",")) + + var sb strings.Builder + + sb.WriteString("for doc in [") + for i, val := range listOfValues { + bs, _ := json.Marshal(val) + if i == len(listOfValues)-1 { + sb.WriteString(string(bs)) + } else { + sb.WriteString(string(bs) + ",") + } + } + sb.WriteString("]") + + if pkgMatchType.Pkg == model.PkgMatchTypeSpecificVersion { + query := ` + LET firstPkg = FIRST( + FOR pVersion in pkgVersions + FILTER pVersion.guacKey == doc.pkgVersionGuacKey + FOR pName in pkgNames + FILTER pName._id == pVersion._parent + FOR pNs in pkgNamespaces + FILTER pNs._id == pName._parent + FOR pType in pkgTypes + FILTER pType._id == pNs._parent + + RETURN { + 'typeID': pType._id, + 'type': pType.type, + 'namespace_id': pNs._id, + 'namespace': pNs.namespace, + 'name_id': pName._id, + 'name': pName.name, + 'version_id': pVersion._id, + 'version': pVersion.version, + 'subpath': pVersion.subpath, + 'qualifier_list': pVersion.qualifier_list, + 'versionDoc': pVersion + } + ) + + LET certifyGood = FIRST( + UPSERT { packageID:firstPkg.version_id, justification:doc.justification, collector:doc.collector, origin:doc.origin } + INSERT { packageID:firstPkg.version_id, justification:doc.justification, collector:doc.collector, origin:doc.origin } + UPDATE {} IN certifyGoods + RETURN NEW + ) + + LET edgeCollection = ( + INSERT { _key: CONCAT("certifyGoodPkgVersionEdges", firstPkg.versionDoc._key, certifyGood._key), _from: firstPkg.version_id, _to: certifyGood._id } INTO certifyGoodPkgVersionEdges OPTIONS { overwriteMode: "ignore" } + ) + + RETURN { + 'pkgVersion': { + 'type_id': firstPkg.typeID, + 'type': firstPkg.type, + 'namespace_id': firstPkg.namespace_id, + 'namespace': firstPkg.namespace, + 'name_id': firstPkg.name_id, + 'name': firstPkg.name, + 'version_id': firstPkg.version_id, + 'version': firstPkg.version, + 'subpath': firstPkg.subpath, + 'qualifier_list': firstPkg.qualifier_list + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }` + + sb.WriteString(query) + + cursor, err := executeQueryWithRetry(ctx, c.db, sb.String(), nil, "IngestCertifyGoods - PkgVersion") + if err != nil { + return nil, fmt.Errorf("failed to ingest package certifyGoods: %w", err) + } + defer cursor.Close() + + certifyGoodList, err := getPkgVersionCertifyGood(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("failed to get certifyGoods from arango cursor: %w", err) + } + + return certifyGoodList, nil + } else { + query := ` + LET firstPkg = FIRST( + FOR pName in pkgNames + FILTER pName.guacKey == doc.pkgNameGuacKey + FOR pNs in pkgNamespaces + FILTER pNs._id == pName._parent + FOR pType in pkgTypes + FILTER pType._id == pNs._parent + + RETURN { + 'typeID': pType._id, + 'type': pType.type, + 'namespace_id': pNs._id, + 'namespace': pNs.namespace, + 'name_id': pName._id, + 'name': pName.name, + 'nameDoc': pName + } + ) + + LET certifyGood = FIRST( + UPSERT { packageID:firstPkg.name_id, justification:doc.justification, collector:doc.collector, origin:doc.origin } + INSERT { packageID:firstPkg.name_id, justification:doc.justification, collector:doc.collector, origin:doc.origin } + UPDATE {} IN certifyGoods + RETURN NEW + ) + + LET edgeCollection = ( + INSERT { _key: CONCAT("certifyGoodPkgNameEdges", firstPkg.nameDoc._key, certifyGood._key), _from: firstPkg.name_id, _to: certifyGood._id } INTO certifyGoodPkgNameEdges OPTIONS { overwriteMode: "ignore" } + ) + + RETURN { + 'pkgName': { + 'type_id': firstPkg.typeID, + 'type': firstPkg.type, + 'namespace_id': firstPkg.namespace_id, + 'namespace': firstPkg.namespace, + 'name_id': firstPkg.name_id, + 'name': firstPkg.name + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }` + + sb.WriteString(query) + + cursor, err := executeQueryWithRetry(ctx, c.db, sb.String(), nil, "IngestCertifyGoods - PkgName") + if err != nil { + return nil, fmt.Errorf("failed to ingest package certifyGoods: %w", err) + } + defer cursor.Close() + + certifyGoodList, err := getPkgNameCertifyGood(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("failed to get certifyGoods from arango cursor: %w", err) + } + + return certifyGoodList, nil + } + + } else if len(subjects.Artifacts) > 0 { + + if len(subjects.Artifacts) != len(certifyGoods) { + return nil, fmt.Errorf("uneven artifacts and certifyGoods for ingestion") + } + + var listOfValues []map[string]any + + for i := range subjects.Artifacts { + listOfValues = append(listOfValues, getCertifyGoodQueryValues(nil, nil, subjects.Artifacts[i], nil, certifyGoods[i])) + } + + var documents []string + for _, val := range listOfValues { + bs, _ := json.Marshal(val) + documents = append(documents, string(bs)) + } + + queryValues := map[string]any{} + queryValues["documents"] = fmt.Sprint(strings.Join(documents, ",")) + + var sb strings.Builder + + sb.WriteString("for doc in [") + for i, val := range listOfValues { + bs, _ := json.Marshal(val) + if i == len(listOfValues)-1 { + sb.WriteString(string(bs)) + } else { + sb.WriteString(string(bs) + ",") + } + } + sb.WriteString("]") + + query := `LET artifact = FIRST(FOR art IN artifacts FILTER art.algorithm == doc.art_algorithm FILTER art.digest == doc.art_digest RETURN art) + + LET certifyGood = FIRST( + UPSERT { artifactID:artifact._id, justification:doc.justification, collector:doc.collector, origin:doc.origin } + INSERT { artifactID:artifact._id, justification:doc.justification, collector:doc.collector, origin:doc.origin } + UPDATE {} IN certifyGoods + RETURN NEW + ) + + LET edgeCollection = ( + INSERT { _key: CONCAT("certifyGoodArtEdges", artifact._key, certifyGood._key), _from: artifact._id, _to: certifyGood._id } INTO certifyGoodArtEdges OPTIONS { overwriteMode: "ignore" } + ) + + RETURN { + 'artifact': { + 'id': artifact._id, + 'algorithm': artifact.algorithm, + 'digest': artifact.digest + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }` + + sb.WriteString(query) + + cursor, err := executeQueryWithRetry(ctx, c.db, sb.String(), nil, "IngestCertifyGoods - artifact") + if err != nil { + return nil, fmt.Errorf("failed to ingest artifact certifyGoods %w", err) + } + defer cursor.Close() + certifyGoodList, err := getArtifactCertifyGood(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("failed to get certifyGoods from arango cursor: %w", err) + } + + return certifyGoodList, nil + + } else if len(subjects.Sources) > 0 { + + if len(subjects.Sources) != len(certifyGoods) { + return nil, fmt.Errorf("uneven sources and certifyGoods for ingestion") + } + + var listOfValues []map[string]any + + for i := range subjects.Sources { + listOfValues = append(listOfValues, getCertifyGoodQueryValues(nil, nil, nil, subjects.Sources[i], certifyGoods[i])) + } + + var documents []string + for _, val := range listOfValues { + bs, _ := json.Marshal(val) + documents = append(documents, string(bs)) + } + + queryValues := map[string]any{} + queryValues["documents"] = fmt.Sprint(strings.Join(documents, ",")) + + var sb strings.Builder + + sb.WriteString("for doc in [") + for i, val := range listOfValues { + bs, _ := json.Marshal(val) + if i == len(listOfValues)-1 { + sb.WriteString(string(bs)) + } else { + sb.WriteString(string(bs) + ",") + } + } + sb.WriteString("]") + + query := ` + LET firstSrc = FIRST( + FOR sName in srcNames + FILTER sName.guacKey == doc.srcNameGuacKey + FOR sNs in srcNamespaces + FILTER sNs._id == sName._parent + FOR sType in srcTypes + FILTER sType._id == sNs._parent + + RETURN { + 'typeID': sType._id, + 'type': sType.type, + 'namespace_id': sNs._id, + 'namespace': sNs.namespace, + 'name_id': sName._id, + 'name': sName.name, + 'commit': sName.commit, + 'tag': sName.tag, + 'nameDoc': sName + } + ) + + LET certifyGood = FIRST( + UPSERT { sourceID:firstSrc.name_id, justification:doc.justification, collector:doc.collector, origin:doc.origin } + INSERT { sourceID:firstSrc.name_id, justification:doc.justification, collector:doc.collector, origin:doc.origin } + UPDATE {} IN certifyGoods + RETURN NEW + ) + + LET edgeCollection = ( + INSERT { _key: CONCAT("certifyGoodSrcEdges", firstSrc.nameDoc._key, certifyGood._key), _from: firstSrc.name_id, _to: certifyGood._id } INTO certifyGoodSrcEdges OPTIONS { overwriteMode: "ignore" } + ) + + RETURN { + 'srcName': { + 'type_id': firstSrc.typeID, + 'type': firstSrc.type, + 'namespace_id': firstSrc.namespace_id, + 'namespace': firstSrc.namespace, + 'name_id': firstSrc.name_id, + 'name': firstSrc.name, + 'commit': firstSrc.commit, + 'tag': firstSrc.tag + }, + 'certifyGood_id': certifyGood._id, + 'justification': certifyGood.justification, + 'collector': certifyGood.collector, + 'origin': certifyGood.origin + }` + + sb.WriteString(query) + + cursor, err := executeQueryWithRetry(ctx, c.db, sb.String(), nil, "IngestCertifyGoods - source") + if err != nil { + return nil, fmt.Errorf("failed to ingest source certifyGoods: %w", err) + } + defer cursor.Close() + certifyGoodList, err := getSourceCertifyGood(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("failed to get certifyGoods from arango cursor: %w", err) + } + + return certifyGoodList, nil + + } else { + return nil, fmt.Errorf("packages, artifacts, or sources not specified for IngestCertifyGoods") + } +} + +func getPkgNameCertifyGood(ctx context.Context, cursor driver.Cursor) ([]*model.CertifyGood, error) { + type collectedData struct { + PkgName *dbPkgName `json:"pkgName"` + CertifyGoodID string `json:"certifyGood_id"` + Justification string `json:"justification"` + Collector string `json:"collector"` + Origin string `json:"origin"` + } + + var createdValues []collectedData + for { + var doc collectedData + _, err := cursor.ReadDocument(ctx, &doc) + if err != nil { + if driver.IsNoMoreDocuments(err) { + break + } else { + return nil, fmt.Errorf("failed to package name certifyGood from cursor: %w", err) + } + } else { + createdValues = append(createdValues, doc) + } + } + + var certifyGoodList []*model.CertifyGood + for _, createdValue := range createdValues { + pkg := generateModelPackage(createdValue.PkgName.TypeID, createdValue.PkgName.PkgType, createdValue.PkgName.NamespaceID, createdValue.PkgName.Namespace, createdValue.PkgName.NameID, + createdValue.PkgName.Name, nil, nil, nil, nil) + + certifyGood := &model.CertifyGood{ + ID: createdValue.CertifyGoodID, + Subject: pkg, + Justification: createdValue.Justification, + Origin: createdValue.Collector, + Collector: createdValue.Origin, + } + certifyGoodList = append(certifyGoodList, certifyGood) + } + return certifyGoodList, nil +} + +func getPkgVersionCertifyGood(ctx context.Context, cursor driver.Cursor) ([]*model.CertifyGood, error) { + type collectedData struct { + PkgVersion *dbPkgVersion `json:"pkgVersion"` + CertifyGoodID string `json:"certifyGood_id"` + Justification string `json:"justification"` + Collector string `json:"collector"` + Origin string `json:"origin"` + } + + var createdValues []collectedData + for { + var doc collectedData + _, err := cursor.ReadDocument(ctx, &doc) + if err != nil { + if driver.IsNoMoreDocuments(err) { + break + } else { + return nil, fmt.Errorf("failed to package version certifyGood from cursor: %w", err) + } + } else { + createdValues = append(createdValues, doc) + } + } + + var certifyGoodList []*model.CertifyGood + for _, createdValue := range createdValues { + pkg := generateModelPackage(createdValue.PkgVersion.TypeID, createdValue.PkgVersion.PkgType, createdValue.PkgVersion.NamespaceID, createdValue.PkgVersion.Namespace, createdValue.PkgVersion.NameID, + createdValue.PkgVersion.Name, &createdValue.PkgVersion.VersionID, &createdValue.PkgVersion.Version, &createdValue.PkgVersion.Subpath, createdValue.PkgVersion.QualifierList) + + certifyGood := &model.CertifyGood{ + ID: createdValue.CertifyGoodID, + Subject: pkg, + Justification: createdValue.Justification, + Origin: createdValue.Collector, + Collector: createdValue.Origin, + } + certifyGoodList = append(certifyGoodList, certifyGood) + } + return certifyGoodList, nil +} + +func getArtifactCertifyGood(ctx context.Context, cursor driver.Cursor) ([]*model.CertifyGood, error) { + type collectedData struct { + Artifact *model.Artifact `json:"artifact"` + CertifyGoodID string `json:"certifyGood_id"` + Justification string `json:"justification"` + Collector string `json:"collector"` + Origin string `json:"origin"` + } + + var createdValues []collectedData + for { + var doc collectedData + _, err := cursor.ReadDocument(ctx, &doc) + if err != nil { + if driver.IsNoMoreDocuments(err) { + break + } else { + return nil, fmt.Errorf("failed to artifact certifyGood from cursor: %w", err) + } + } else { + createdValues = append(createdValues, doc) + } + } + + var certifyGoodList []*model.CertifyGood + for _, createdValue := range createdValues { + certifyGood := &model.CertifyGood{ + ID: createdValue.CertifyGoodID, + Subject: createdValue.Artifact, + Justification: createdValue.Justification, + Origin: createdValue.Collector, + Collector: createdValue.Origin, + } + certifyGoodList = append(certifyGoodList, certifyGood) + } + return certifyGoodList, nil +} + +func getSourceCertifyGood(ctx context.Context, cursor driver.Cursor) ([]*model.CertifyGood, error) { + type collectedData struct { + SrcName *dbSrcName `json:"srcName"` + CertifyGoodID string `json:"certifyGood_id"` + Justification string `json:"justification"` + Collector string `json:"collector"` + Origin string `json:"origin"` + } + + var createdValues []collectedData + for { + var doc collectedData + _, err := cursor.ReadDocument(ctx, &doc) + if err != nil { + if driver.IsNoMoreDocuments(err) { + break + } else { + return nil, fmt.Errorf("failed to source certifyGood from cursor: %w", err) + } + } else { + createdValues = append(createdValues, doc) + } + } + + var certifyGoodList []*model.CertifyGood + for _, createdValue := range createdValues { + + src := generateModelSource(createdValue.SrcName.TypeID, createdValue.SrcName.SrcType, createdValue.SrcName.NamespaceID, createdValue.SrcName.Namespace, + createdValue.SrcName.NameID, createdValue.SrcName.Name, createdValue.SrcName.Commit, createdValue.SrcName.Tag) + + certifyGood := &model.CertifyGood{ + ID: createdValue.CertifyGoodID, + Subject: src, + Justification: createdValue.Justification, + Origin: createdValue.Collector, + Collector: createdValue.Origin, + } + certifyGoodList = append(certifyGoodList, certifyGood) + } + return certifyGoodList, nil } diff --git a/pkg/assembler/backends/arangodb/certifyScorecard.go b/pkg/assembler/backends/arangodb/certifyScorecard.go index d8fabe1ea0..b95c028e15 100644 --- a/pkg/assembler/backends/arangodb/certifyScorecard.go +++ b/pkg/assembler/backends/arangodb/certifyScorecard.go @@ -85,6 +85,10 @@ func (c *arangoClient) Scorecards(ctx context.Context, certifyScorecardSpec *mod } func setCertifyScorecardMatchValues(arangoQueryBuilder *arangoQueryBuilder, certifyScorecardSpec *model.CertifyScorecardSpec, queryValues map[string]any) { + if certifyScorecardSpec.ID != nil { + arangoQueryBuilder.filter("scorecard", "_id", "==", "@id") + queryValues["id"] = *certifyScorecardSpec.ID + } if certifyScorecardSpec.TimeScanned != nil { arangoQueryBuilder.filter("scorecard", timeScannedStr, "==", "@"+timeScannedStr) queryValues[timeScannedStr] = certifyScorecardSpec.TimeScanned.UTC() diff --git a/pkg/assembler/backends/arangodb/cve.go b/pkg/assembler/backends/arangodb/cve.go index f67585af79..f6f663d2e8 100644 --- a/pkg/assembler/backends/arangodb/cve.go +++ b/pkg/assembler/backends/arangodb/cve.go @@ -28,6 +28,10 @@ import ( func (c *arangoClient) Cve(ctx context.Context, cveSpec *model.CVESpec) ([]*model.Cve, error) { values := map[string]any{} arangoQueryBuilder := newForQuery(cvesStr, "cve") + if cveSpec.ID != nil { + arangoQueryBuilder.filter("cve", "_id", "==", "@id") + values["id"] = *cveSpec.ID + } if cveSpec.Year != nil { arangoQueryBuilder.filter("cve", "year", "==", "@year") values["year"] = *cveSpec.Year diff --git a/pkg/assembler/backends/arangodb/ghsa.go b/pkg/assembler/backends/arangodb/ghsa.go index 15dfe2df5a..0e0ea6f674 100644 --- a/pkg/assembler/backends/arangodb/ghsa.go +++ b/pkg/assembler/backends/arangodb/ghsa.go @@ -28,6 +28,10 @@ import ( func (c *arangoClient) Ghsa(ctx context.Context, ghsaSpec *model.GHSASpec) ([]*model.Ghsa, error) { values := map[string]any{} arangoQueryBuilder := newForQuery(ghsasStr, "ghsa") + if ghsaSpec.ID != nil { + arangoQueryBuilder.filter("ghsa", "_id", "==", "@id") + values["id"] = *ghsaSpec.ID + } if ghsaSpec.GhsaID != nil { arangoQueryBuilder.filter("ghsa", "ghsaId", "==", "@ghsaId") values["ghsaId"] = strings.ToLower(*ghsaSpec.GhsaID) diff --git a/pkg/assembler/backends/arangodb/hasSBOM.go b/pkg/assembler/backends/arangodb/hasSBOM.go index 21a62eb347..eabeb521b7 100644 --- a/pkg/assembler/backends/arangodb/hasSBOM.go +++ b/pkg/assembler/backends/arangodb/hasSBOM.go @@ -28,29 +28,43 @@ import ( func (c *arangoClient) HasSBOM(ctx context.Context, hasSBOMSpec *model.HasSBOMSpec) ([]*model.HasSbom, error) { // TODO (pxp928): Optimize/add other queries based on input and starting node/edge for most efficient retrieval - values := map[string]any{} var arangoQueryBuilder *arangoQueryBuilder if hasSBOMSpec.Subject != nil { + var combinedHasSBOM []*model.HasSbom if hasSBOMSpec.Subject.Package != nil { - arangoQueryBuilder = setPkgMatchValues(hasSBOMSpec.Subject.Package, values) + values := map[string]any{} + arangoQueryBuilder = setPkgVersionMatchValues(hasSBOMSpec.Subject.Package, values) arangoQueryBuilder.forOutBound(hasSBOMPkgEdgesStr, "hasSBOM", "pVersion") setHasSBOMMatchValues(arangoQueryBuilder, hasSBOMSpec, values) - return getPkgHasSBOMForQuery(ctx, c, arangoQueryBuilder, values) - } else { - arangoQueryBuilder = setArtifactMatchValues(nil, hasSBOMSpec.Subject.Artifact, values) + pkgVersionHasSboms, err := getPkgHasSBOMForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package version hasSBOM with error: %w", err) + } + + combinedHasSBOM = append(combinedHasSBOM, pkgVersionHasSboms...) + } + if hasSBOMSpec.Subject.Artifact != nil { + values := map[string]any{} + arangoQueryBuilder = setArtifactMatchValues(hasSBOMSpec.Subject.Artifact, values) arangoQueryBuilder.forOutBound(hasSBOMArtEdgesStr, "hasSBOM", "art") setHasSBOMMatchValues(arangoQueryBuilder, hasSBOMSpec, values) - return getArtifactHasSBOMForQuery(ctx, c, arangoQueryBuilder, values) + artHasSboms, err := getArtifactHasSBOMForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve artifact hasSBOM with error: %w", err) + } + combinedHasSBOM = append(combinedHasSBOM, artHasSboms...) } + return combinedHasSBOM, nil } else { + values := map[string]any{} var combinedHasSBOM []*model.HasSbom // get packages arangoQueryBuilder = newForQuery(hasSBOMsStr, "hasSBOM") setHasSBOMMatchValues(arangoQueryBuilder, hasSBOMSpec, values) - arangoQueryBuilder.forInBoundWithEdgeCounter(hasSBOMPkgEdgesStr, "pVersion", "hasSBOMEdge", "hasSBOM") + arangoQueryBuilder.forInBound(hasSBOMPkgEdgesStr, "pVersion", "hasSBOM") arangoQueryBuilder.forInBound(pkgHasVersionStr, "pName", "pVersion") arangoQueryBuilder.forInBound(pkgHasNameStr, "pNs", "pName") arangoQueryBuilder.forInBound(pkgHasNamespaceStr, "pType", "pNs") @@ -64,7 +78,7 @@ func (c *arangoClient) HasSBOM(ctx context.Context, hasSBOMSpec *model.HasSBOMSp // get artifacts arangoQueryBuilder = newForQuery(hasSBOMsStr, "hasSBOM") setHasSBOMMatchValues(arangoQueryBuilder, hasSBOMSpec, values) - arangoQueryBuilder.forInBoundWithEdgeCounter(hasSBOMArtEdgesStr, "art", "hasSBOMEdge", "hasSBOM") + arangoQueryBuilder.forInBound(hasSBOMArtEdgesStr, "art", "hasSBOM") artifactHasSBOMs, err := getArtifactHasSBOMForQuery(ctx, c, arangoQueryBuilder, values) if err != nil { @@ -140,6 +154,10 @@ func getArtifactHasSBOMForQuery(ctx context.Context, c *arangoClient, arangoQuer } func setHasSBOMMatchValues(arangoQueryBuilder *arangoQueryBuilder, hasSBOMSpec *model.HasSBOMSpec, queryValues map[string]any) { + if hasSBOMSpec.ID != nil { + arangoQueryBuilder.filter("hasSBOM", "_id", "==", "@id") + queryValues["id"] = *hasSBOMSpec.ID + } if hasSBOMSpec.URI != nil { arangoQueryBuilder.filter("hasSBOM", "uri", "==", "@uri") queryValues["uri"] = hasSBOMSpec.URI diff --git a/pkg/assembler/backends/arangodb/hasSLSA.go b/pkg/assembler/backends/arangodb/hasSLSA.go index 51702f6cc3..2afb54e250 100644 --- a/pkg/assembler/backends/arangodb/hasSLSA.go +++ b/pkg/assembler/backends/arangodb/hasSLSA.go @@ -38,9 +38,9 @@ const ( func (c *arangoClient) HasSlsa(ctx context.Context, hasSLSASpec *model.HasSLSASpec) ([]*model.HasSlsa, error) { - // TODO (pxp928): Optimize/add other queries based on input and starting node/edge for most efficient retrieval + // TODO (pxp928): Optimize/add other queries based on input and starting node/edge for most efficient retrieval (like from builtBy/builtFrom if specified) values := map[string]any{} - arangoQueryBuilder := setArtifactMatchValues(nil, hasSLSASpec.Subject, values) + arangoQueryBuilder := setArtifactMatchValues(hasSLSASpec.Subject, values) setHasSLSAMatchValues(arangoQueryBuilder, hasSLSASpec, values) arangoQueryBuilder.query.WriteString("\n") @@ -80,6 +80,10 @@ func setHasSLSAMatchValues(arangoQueryBuilder *arangoQueryBuilder, hasSLSASpec * // currently not filtering on builtFrom (artifacts). Is that a real usecase? arangoQueryBuilder.forOutBound(hasSLSASubjectArtEdgesStr, "hasSLSA", "art") + if hasSLSASpec.ID != nil { + arangoQueryBuilder.filter("hasSLSA", "_id", "==", "@id") + queryValues["id"] = *hasSLSASpec.ID + } if hasSLSASpec.BuildType != nil { arangoQueryBuilder.filter("hasSLSA", buildTypeStr, "==", "@"+buildTypeStr) queryValues[buildTypeStr] = hasSLSASpec.BuildType diff --git a/pkg/assembler/backends/arangodb/hashEqual.go b/pkg/assembler/backends/arangodb/hashEqual.go index c39871bb81..474ea2c717 100644 --- a/pkg/assembler/backends/arangodb/hashEqual.go +++ b/pkg/assembler/backends/arangodb/hashEqual.go @@ -51,7 +51,7 @@ func matchHashEqualByInput(ctx context.Context, c *arangoClient, hashEqualSpec * var combinedHashEqual []*model.HashEqual - arangoQueryBuilder := setArtifactMatchValues(nil, firstArtifact, values) + arangoQueryBuilder := setArtifactMatchValues(firstArtifact, values) arangoQueryBuilder.forOutBound(hashEqualSubjectArtEdgesStr, "hashEqual", "art") setHashEqualMatchValues(arangoQueryBuilder, hashEqualSpec, values) arangoQueryBuilder.forOutBound(hashEqualArtEdgesStr, "equalArt", "hashEqual") @@ -76,7 +76,7 @@ func matchHashEqualByInput(ctx context.Context, c *arangoClient, hashEqualSpec * } combinedHashEqual = append(combinedHashEqual, artSubjectHashEqual...) - arangoQueryBuilder = setArtifactMatchValues(nil, firstArtifact, values) + arangoQueryBuilder = setArtifactMatchValues(firstArtifact, values) arangoQueryBuilder.forInBound(hashEqualArtEdgesStr, "hashEqual", "art") setHashEqualMatchValues(arangoQueryBuilder, hashEqualSpec, values) arangoQueryBuilder.forInBound(hashEqualSubjectArtEdgesStr, "equalArt", "hashEqual") @@ -135,6 +135,10 @@ func getHashEqualForQuery(ctx context.Context, c *arangoClient, arangoQueryBuild } func setHashEqualMatchValues(arangoQueryBuilder *arangoQueryBuilder, hashEqualSpec *model.HashEqualSpec, queryValues map[string]any) { + if hashEqualSpec.ID != nil { + arangoQueryBuilder.filter("hashEqual", "_id", "==", "@id") + queryValues["id"] = *hashEqualSpec.ID + } if hashEqualSpec.Justification != nil { arangoQueryBuilder.filter("hashEqual", justification, "==", "@"+justification) queryValues[justification] = hashEqualSpec.Justification diff --git a/pkg/assembler/backends/arangodb/isDependency.go b/pkg/assembler/backends/arangodb/isDependency.go index 6f93cad47b..736f9ed64d 100644 --- a/pkg/assembler/backends/arangodb/isDependency.go +++ b/pkg/assembler/backends/arangodb/isDependency.go @@ -34,11 +34,31 @@ const ( func (c *arangoClient) IsDependency(ctx context.Context, isDependencySpec *model.IsDependencySpec) ([]*model.IsDependency, error) { - // TODO (pxp928): Optimize/add other queries based on input and starting node/edge for most efficient retrieval - values := map[string]any{} - arangoQueryBuilder := setPkgMatchValues(isDependencySpec.Package, values) - setIsDependencyMatchValues(arangoQueryBuilder, isDependencySpec, values) + // TODO (pxp928): Optimization of the query can be done by starting from the dependent package node (if specified) + var arangoQueryBuilder *arangoQueryBuilder + + if isDependencySpec.Package != nil { + values := map[string]any{} + arangoQueryBuilder := setPkgVersionMatchValues(isDependencySpec.Package, values) + arangoQueryBuilder.forOutBound(isDependencySubjectPkgEdgesStr, "isDependency", "pVersion") + setIsDependencyMatchValues(arangoQueryBuilder, isDependencySpec, values) + + return getDependencyForQuery(ctx, c, arangoQueryBuilder, values) + } else { + values := map[string]any{} + // get packages + arangoQueryBuilder = newForQuery(isDependenciesStr, "isDependency") + setIsDependencyMatchValues(arangoQueryBuilder, isDependencySpec, values) + arangoQueryBuilder.forInBound(isDependencySubjectPkgEdgesStr, "pVersion", "isDependency") + arangoQueryBuilder.forInBound(pkgHasVersionStr, "pName", "pVersion") + arangoQueryBuilder.forInBound(pkgHasNameStr, "pNs", "pName") + arangoQueryBuilder.forInBound(pkgHasNamespaceStr, "pType", "pNs") + + return getDependencyForQuery(ctx, c, arangoQueryBuilder, values) + } +} +func getDependencyForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.IsDependency, error) { arangoQueryBuilder.query.WriteString("\n") arangoQueryBuilder.query.WriteString(`RETURN { 'pkgVersion': { @@ -81,8 +101,10 @@ func (c *arangoClient) IsDependency(ctx context.Context, isDependencySpec *model } func setIsDependencyMatchValues(arangoQueryBuilder *arangoQueryBuilder, isDependencySpec *model.IsDependencySpec, queryValues map[string]any) { - - arangoQueryBuilder.forOutBound(isDependencySubjectPkgEdgesStr, "isDependency", "pVersion") + if isDependencySpec.ID != nil { + arangoQueryBuilder.filter("isDependency", "_id", "==", "@id") + queryValues["id"] = *isDependencySpec.ID + } if isDependencySpec.VersionRange != nil { arangoQueryBuilder.filter("isDependency", versionRangeStr, "==", "@"+versionRangeStr) queryValues[versionRangeStr] = isDependencySpec.VersionRange diff --git a/pkg/assembler/backends/arangodb/isOccurrence.go b/pkg/assembler/backends/arangodb/isOccurrence.go index 30e1d2ff93..3513f40e38 100644 --- a/pkg/assembler/backends/arangodb/isOccurrence.go +++ b/pkg/assembler/backends/arangodb/isOccurrence.go @@ -28,29 +28,46 @@ import ( // Query IsOccurrence func (c *arangoClient) IsOccurrence(ctx context.Context, isOccurrenceSpec *model.IsOccurrenceSpec) ([]*model.IsOccurrence, error) { - // TODO (pxp928): Optimize/add other queries based on input and starting node/edge for most efficient retrieval - values := map[string]any{} + // TODO (pxp928): Optimization of the query can be done by starting from the occurrence artifact node (if specified) var arangoQueryBuilder *arangoQueryBuilder if isOccurrenceSpec.Subject != nil { + var combinedOccurrence []*model.IsOccurrence if isOccurrenceSpec.Subject.Package != nil { - arangoQueryBuilder = setPkgMatchValues(isOccurrenceSpec.Subject.Package, values) + values := map[string]any{} + + arangoQueryBuilder = setPkgVersionMatchValues(isOccurrenceSpec.Subject.Package, values) arangoQueryBuilder.forOutBound(isOccurrenceSubjectPkgEdgesStr, "isOccurrence", "pVersion") setIsOccurrenceMatchValues(arangoQueryBuilder, isOccurrenceSpec, values) - return getPkgOccurrencesForQuery(ctx, c, arangoQueryBuilder, values) - } else { + pkgVersionOccurrences, err := getPkgOccurrencesForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve package version occurrences with error: %w", err) + } + + combinedOccurrence = append(combinedOccurrence, pkgVersionOccurrences...) + } + if isOccurrenceSpec.Subject.Source != nil { + values := map[string]any{} + arangoQueryBuilder = setSrcMatchValues(isOccurrenceSpec.Subject.Source, values) arangoQueryBuilder.forOutBound(isOccurrenceSubjectSrcEdgesStr, "isOccurrence", "sName") setIsOccurrenceMatchValues(arangoQueryBuilder, isOccurrenceSpec, values) - return getSrcOccurrencesForQuery(ctx, c, arangoQueryBuilder, values) + srcOccurrences, err := getSrcOccurrencesForQuery(ctx, c, arangoQueryBuilder, values) + if err != nil { + return nil, fmt.Errorf("failed to retrieve source occurrences with error: %w", err) + } + + combinedOccurrence = append(combinedOccurrence, srcOccurrences...) } + return combinedOccurrence, nil } else { var combinedOccurrence []*model.IsOccurrence + values := map[string]any{} // get packages arangoQueryBuilder = newForQuery(isOccurrencesStr, "isOccurrence") setIsOccurrenceMatchValues(arangoQueryBuilder, isOccurrenceSpec, values) - arangoQueryBuilder.forInBoundWithEdgeCounter(isOccurrenceSubjectPkgEdgesStr, "pVersion", "isOccurEdgePkg", "isOccurrence") + arangoQueryBuilder.forInBound(isOccurrenceSubjectPkgEdgesStr, "pVersion", "isOccurrence") arangoQueryBuilder.forInBound(pkgHasVersionStr, "pName", "pVersion") arangoQueryBuilder.forInBound(pkgHasNameStr, "pNs", "pName") arangoQueryBuilder.forInBound(pkgHasNamespaceStr, "pType", "pNs") @@ -64,7 +81,7 @@ func (c *arangoClient) IsOccurrence(ctx context.Context, isOccurrenceSpec *model // get sources arangoQueryBuilder = newForQuery(isOccurrencesStr, "isOccurrence") setIsOccurrenceMatchValues(arangoQueryBuilder, isOccurrenceSpec, values) - arangoQueryBuilder.forInBoundWithEdgeCounter(isOccurrenceSubjectSrcEdgesStr, "sName", "isOccurEdgeSrc", "isOccurrence") + arangoQueryBuilder.forInBound(isOccurrenceSubjectSrcEdgesStr, "sName", "isOccurrence") arangoQueryBuilder.forInBound(srcHasNameStr, "sNs", "sName") arangoQueryBuilder.forInBound(srcHasNamespaceStr, "sType", "sNs") @@ -82,7 +99,7 @@ func getSrcOccurrencesForQuery(ctx context.Context, c *arangoClient, arangoQuery arangoQueryBuilder.query.WriteString("\n") arangoQueryBuilder.query.WriteString(`RETURN { 'srcName': { - 'typeID': sType._id, + 'type_id': sType._id, 'type': sType.type, 'namespace_id': sNs._id, 'namespace': sNs.namespace, @@ -117,7 +134,7 @@ func getPkgOccurrencesForQuery(ctx context.Context, c *arangoClient, arangoQuery arangoQueryBuilder.query.WriteString("\n") arangoQueryBuilder.query.WriteString(`RETURN { 'pkgVersion': { - 'typeID': pType._id, + 'type_id': pType._id, 'type': pType.type, 'namespace_id': pNs._id, 'namespace': pNs.namespace, @@ -151,6 +168,10 @@ func getPkgOccurrencesForQuery(ctx context.Context, c *arangoClient, arangoQuery } func setIsOccurrenceMatchValues(arangoQueryBuilder *arangoQueryBuilder, isOccurrenceSpec *model.IsOccurrenceSpec, queryValues map[string]any) { + if isOccurrenceSpec.ID != nil { + arangoQueryBuilder.filter("isOccurrence", "_id", "==", "@id") + queryValues["id"] = *isOccurrenceSpec.ID + } if isOccurrenceSpec.Justification != nil { arangoQueryBuilder.filter("isOccurrence", justification, "==", "@"+justification) queryValues[justification] = isOccurrenceSpec.Justification @@ -165,7 +186,18 @@ func setIsOccurrenceMatchValues(arangoQueryBuilder *arangoQueryBuilder, isOccurr } arangoQueryBuilder.forOutBound(isOccurrenceArtEdgesStr, "art", "isOccurrence") if isOccurrenceSpec.Artifact != nil { - setArtifactMatchValues(arangoQueryBuilder, isOccurrenceSpec.Artifact, queryValues) + if isOccurrenceSpec.Artifact.ID != nil { + arangoQueryBuilder.filter("art", "_id", "==", "@id") + queryValues["id"] = *isOccurrenceSpec.Artifact.ID + } + if isOccurrenceSpec.Artifact.Algorithm != nil { + arangoQueryBuilder.filter("art", "algorithm", "==", "@algorithm") + queryValues["algorithm"] = strings.ToLower(*isOccurrenceSpec.Artifact.Algorithm) + } + if isOccurrenceSpec.Artifact.Digest != nil { + arangoQueryBuilder.filter("art", "digest", "==", "@digest") + queryValues["digest"] = strings.ToLower(*isOccurrenceSpec.Artifact.Digest) + } } } diff --git a/pkg/assembler/backends/arangodb/osv.go b/pkg/assembler/backends/arangodb/osv.go index fe8f4c2c25..c2355ec832 100644 --- a/pkg/assembler/backends/arangodb/osv.go +++ b/pkg/assembler/backends/arangodb/osv.go @@ -28,6 +28,10 @@ import ( func (c *arangoClient) Osv(ctx context.Context, osvSpec *model.OSVSpec) ([]*model.Osv, error) { values := map[string]any{} arangoQueryBuilder := newForQuery(osvsStr, "osv") + if osvSpec.ID != nil { + arangoQueryBuilder.filter("osv", "_id", "==", "@id") + values["id"] = *osvSpec.ID + } if osvSpec.OsvID != nil { arangoQueryBuilder.filter("osv", "osvId", "==", "@osvId") values["osvId"] = strings.ToLower(*osvSpec.OsvID) diff --git a/pkg/assembler/backends/arangodb/pkg.go b/pkg/assembler/backends/arangodb/pkg.go index 952b4a50fb..4aa254247f 100644 --- a/pkg/assembler/backends/arangodb/pkg.go +++ b/pkg/assembler/backends/arangodb/pkg.go @@ -121,10 +121,6 @@ func getPackageQueryValues(c *arangoClient, pkg *model.PkgInputSpec) map[string] values := map[string]any{} // add guac keys - - values["rootID"] = c.pkgRoot.Id - values["rootKey"] = c.pkgRoot.Key - guacIds := guacPkgId(*pkg) values["guacNsKey"] = guacIds.NamespaceId values["guacNameKey"] = guacIds.NameId @@ -194,10 +190,10 @@ func (c *arangoClient) IngestPackages(ctx context.Context, pkgs []*model.PkgInpu query := ` LET type = FIRST( - UPSERT { type: doc.pkgType, _parent: doc.rootID } - INSERT { type: doc.pkgType, _parent: doc.rootID } + UPSERT { type: doc.pkgType} + INSERT { type: doc.pkgType} UPDATE {} - IN pkgTypes OPTIONS { indexHint: "byPkgTypeParent" } + IN pkgTypes OPTIONS { indexHint: "byPkgType" } RETURN NEW ) @@ -224,10 +220,6 @@ func (c *arangoClient) IngestPackages(ctx context.Context, pkgs []*model.PkgInpu IN pkgVersions OPTIONS { indexHint: "byVersionGuacKey" } RETURN NEW ) - - LET pkgHasTypeCollection = ( - INSERT { _key: CONCAT("pkgHasType", doc.rootKey, type._key), _from: doc.rootID, _to: type._id} INTO pkgHasType OPTIONS { overwriteMode: "ignore" } - ) LET pkgHasNamespaceCollection = ( INSERT { _key: CONCAT("pkgHasNamespace", type._key, ns._key), _from: type._id, _to: ns._id } INTO pkgHasNamespace OPTIONS { overwriteMode: "ignore" } @@ -267,10 +259,10 @@ func (c *arangoClient) IngestPackages(ctx context.Context, pkgs []*model.PkgInpu func (c *arangoClient) IngestPackage(ctx context.Context, pkg model.PkgInputSpec) (*model.Package, error) { query := ` LET type = FIRST( - UPSERT { type: @pkgType, _parent: @rootID } - INSERT { type: @pkgType, _parent: @rootID } + UPSERT { type: @pkgType } + INSERT { type: @pkgType } UPDATE {} - IN pkgTypes OPTIONS { indexHint: "byPkgTypeParent" } + IN pkgTypes OPTIONS { indexHint: "byPkgType" } RETURN NEW ) @@ -297,10 +289,6 @@ func (c *arangoClient) IngestPackage(ctx context.Context, pkg model.PkgInputSpec IN pkgVersions OPTIONS { indexHint: "byVersionGuacKey" } RETURN NEW ) - - LET pkgHasTypeCollection = ( - INSERT { _key: CONCAT("pkgHasType", @rootKey, type._key), _from: @rootID, _to: type._id} INTO pkgHasType OPTIONS { overwriteMode: "ignore" } - ) LET pkgHasNamespaceCollection = ( INSERT { _key: CONCAT("pkgHasNamespace", type._key, ns._key), _from: type._id, _to: ns._id} INTO pkgHasNamespace OPTIONS { overwriteMode: "ignore" } @@ -343,13 +331,36 @@ func (c *arangoClient) IngestPackage(ctx context.Context, pkg model.PkgInputSpec } } -func setPkgMatchValues(pkgSpec *model.PkgSpec, queryValues map[string]any) *arangoQueryBuilder { +func setPkgNameMatchValues(pkgSpec *model.PkgSpec, queryValues map[string]any) *arangoQueryBuilder { + var arangoQueryBuilder *arangoQueryBuilder + if pkgSpec != nil { + arangoQueryBuilder = newForQuery(pkgTypesStr, "pType") + if pkgSpec.Type != nil { + arangoQueryBuilder.filter("pType", "type", "==", "@pkgType") + queryValues["pkgType"] = *pkgSpec.Type + } + arangoQueryBuilder.forOutBound(pkgHasNamespaceStr, "pNs", "pType") + if pkgSpec.Namespace != nil { + arangoQueryBuilder.filter("pNs", "namespace", "==", "@namespace") + queryValues["namespace"] = *pkgSpec.Namespace + } + arangoQueryBuilder.forOutBound(pkgHasNameStr, "pName", "pNs") + if pkgSpec.Name != nil { + arangoQueryBuilder.filter("pName", "name", "==", "@name") + queryValues["name"] = *pkgSpec.Name + } + } else { + arangoQueryBuilder = newForQuery(pkgTypesStr, "pType") + arangoQueryBuilder.forOutBound(pkgHasNamespaceStr, "pNs", "pType") + arangoQueryBuilder.forOutBound(pkgHasNameStr, "pName", "pNs") + } + return arangoQueryBuilder +} + +func setPkgVersionMatchValues(pkgSpec *model.PkgSpec, queryValues map[string]any) *arangoQueryBuilder { var arangoQueryBuilder *arangoQueryBuilder if pkgSpec != nil { - arangoQueryBuilder = newForQuery(pkgRootsStr, "pRoot") - arangoQueryBuilder.filter("pRoot", "root", "==", "@pkg") - queryValues["pkg"] = "pkg" - arangoQueryBuilder.forOutBound(pkgHasTypeStr, "pType", "pRoot") + arangoQueryBuilder = newForQuery(pkgTypesStr, "pType") if pkgSpec.Type != nil { arangoQueryBuilder.filter("pType", "type", "==", "@pkgType") queryValues["pkgType"] = *pkgSpec.Type @@ -392,8 +403,7 @@ func setPkgMatchValues(pkgSpec *model.PkgSpec, queryValues map[string]any) *aran } } else { - arangoQueryBuilder = newForQuery(pkgRootsStr, "pRoot") - arangoQueryBuilder.forOutBound(pkgHasTypeStr, "pType", "pRoot") + arangoQueryBuilder = newForQuery(pkgTypesStr, "pType") arangoQueryBuilder.forOutBound(pkgHasNamespaceStr, "pNs", "pType") arangoQueryBuilder.forOutBound(pkgHasNameStr, "pName", "pNs") arangoQueryBuilder.forOutBound(pkgHasVersionStr, "pVersion", "pName") @@ -432,7 +442,7 @@ func (c *arangoClient) Packages(ctx context.Context, pkgSpec *model.PkgSpec) ([] values := map[string]any{} - arangoQueryBuilder := setPkgMatchValues(pkgSpec, values) + arangoQueryBuilder := setPkgVersionMatchValues(pkgSpec, values) arangoQueryBuilder.query.WriteString("\n") arangoQueryBuilder.query.WriteString(`RETURN { "type_id": pType._id, @@ -462,10 +472,7 @@ func (c *arangoClient) packagesType(ctx context.Context, pkgSpec *model.PkgSpec) values := map[string]any{} - arangoQueryBuilder := newForQuery(pkgRootsStr, "pRoot") - arangoQueryBuilder.filter("pRoot", "root", "==", "@pkg") - values["pkg"] = "pkg" - arangoQueryBuilder.forOutBound(pkgHasTypeStr, "pType", "pRoot") + arangoQueryBuilder := newForQuery(pkgTypesStr, "pType") if pkgSpec.Type != nil { arangoQueryBuilder.filter("pType", "type", "==", "@pkgType") values["pkgType"] = *pkgSpec.Type @@ -510,10 +517,7 @@ func (c *arangoClient) packagesType(ctx context.Context, pkgSpec *model.PkgSpec) func (c *arangoClient) packagesNamespace(ctx context.Context, pkgSpec *model.PkgSpec) ([]*model.Package, error) { values := map[string]any{} - arangoQueryBuilder := newForQuery(pkgRootsStr, "pRoot") - arangoQueryBuilder.filter("pRoot", "root", "==", "@pkg") - values["pkg"] = "pkg" - arangoQueryBuilder.forOutBound(pkgHasTypeStr, "pType", "pRoot") + arangoQueryBuilder := newForQuery(pkgTypesStr, "pType") if pkgSpec.Type != nil { arangoQueryBuilder.filter("pType", "type", "==", "@pkgType") values["pkgType"] = *pkgSpec.Type @@ -578,10 +582,7 @@ func (c *arangoClient) packagesNamespace(ctx context.Context, pkgSpec *model.Pkg func (c *arangoClient) packagesName(ctx context.Context, pkgSpec *model.PkgSpec) ([]*model.Package, error) { values := map[string]any{} - arangoQueryBuilder := newForQuery(pkgRootsStr, "pRoot") - arangoQueryBuilder.filter("pRoot", "root", "==", "@pkg") - values["pkg"] = "pkg" - arangoQueryBuilder.forOutBound(pkgHasTypeStr, "pType", "pRoot") + arangoQueryBuilder := newForQuery(pkgTypesStr, "pType") if pkgSpec.Type != nil { arangoQueryBuilder.filter("pType", "type", "==", "@pkgType") values["pkgType"] = *pkgSpec.Type diff --git a/pkg/assembler/backends/arangodb/src.go b/pkg/assembler/backends/arangodb/src.go index 76774f46d7..d853dd8b44 100644 --- a/pkg/assembler/backends/arangodb/src.go +++ b/pkg/assembler/backends/arangodb/src.go @@ -92,9 +92,6 @@ func guacSrcId(src model.SourceInputSpec) SrcIds { func getSourceQueryValues(c *arangoClient, source *model.SourceInputSpec) map[string]any { values := map[string]any{} // add guac keys - values["rootID"] = c.srcRoot.Id - values["rootKey"] = c.srcRoot.Key - guacIds := guacSrcId(*source) values["guacNsKey"] = guacIds.NamespaceId values["guacNameKey"] = guacIds.NameId @@ -149,10 +146,10 @@ func (c *arangoClient) IngestSources(ctx context.Context, sources []*model.Sourc query := ` LET type = FIRST( - UPSERT { type: doc.srcType, _parent: doc.rootID } - INSERT { type: doc.srcType, _parent: doc.rootID } + UPSERT { type: doc.srcType } + INSERT { type: doc.srcType } UPDATE {} - IN srcTypes OPTIONS { indexHint: "byType" } + IN srcTypes OPTIONS { indexHint: "bySrcType" } RETURN NEW ) @@ -172,10 +169,6 @@ func (c *arangoClient) IngestSources(ctx context.Context, sources []*model.Sourc RETURN NEW ) - LET pkgHasTypeCollection = ( - INSERT { _key: CONCAT("srcHasType", doc.rootKey, type._key), _from: doc.rootID, _to: type._id } INTO srcHasType OPTIONS { overwriteMode: "ignore" } - ) - LET srcHasNamespaceCollection = ( INSERT { _key: CONCAT("srcHasNamespace", type._key, ns._key), _from: type._id, _to: ns._id } INTO srcHasNamespace OPTIONS { overwriteMode: "ignore" } ) @@ -208,10 +201,10 @@ func (c *arangoClient) IngestSources(ctx context.Context, sources []*model.Sourc func (c *arangoClient) IngestSource(ctx context.Context, source model.SourceInputSpec) (*model.Source, error) { query := ` LET type = FIRST( - UPSERT { type: @srcType, _parent: @rootID } - INSERT { type: @srcType, _parent: @rootID } + UPSERT { type: @srcType } + INSERT { type: @srcType } UPDATE {} - IN srcTypes OPTIONS { indexHint: "byType" } + IN srcTypes OPTIONS { indexHint: "bySrcType" } RETURN NEW ) @@ -229,11 +222,7 @@ func (c *arangoClient) IngestSource(ctx context.Context, source model.SourceInpu UPDATE {} IN srcNames OPTIONS { indexHint: "byNameGuacKey" } RETURN NEW - ) - - LET pkgHasTypeCollection = ( - INSERT { _key: CONCAT("srcHasType", @rootKey, type._key), _from: @rootID, _to: type._id } INTO srcHasType OPTIONS { overwriteMode: "ignore" } - ) + ) LET srcHasNamespaceCollection = ( INSERT { _key: CONCAT("srcHasNamespace", type._key, ns._key), _from: type._id, _to: ns._id } INTO srcHasNamespace OPTIONS { overwriteMode: "ignore" } @@ -273,10 +262,7 @@ func (c *arangoClient) IngestSource(ctx context.Context, source model.SourceInpu func setSrcMatchValues(srcSpec *model.SourceSpec, queryValues map[string]any) *arangoQueryBuilder { var arangoQueryBuilder *arangoQueryBuilder if srcSpec != nil { - arangoQueryBuilder = newForQuery(srcRootsStr, "sRoot") - arangoQueryBuilder.filter("sRoot", "root", "==", "@src") - queryValues["src"] = "src" - arangoQueryBuilder.forOutBound(srcHasTypeStr, "sType", "sRoot") + arangoQueryBuilder = newForQuery(srcTypesStr, "sType") if srcSpec.Type != nil { arangoQueryBuilder.filter("sType", "type", "==", "@srcType") queryValues["srcType"] = *srcSpec.Type @@ -304,8 +290,7 @@ func setSrcMatchValues(srcSpec *model.SourceSpec, queryValues map[string]any) *a queryValues["tag"] = *srcSpec.Tag } } else { - arangoQueryBuilder = newForQuery(srcRootsStr, "sRoot") - arangoQueryBuilder.forOutBound(srcHasTypeStr, "sType", "sRoot") + arangoQueryBuilder = newForQuery(srcTypesStr, "sType") arangoQueryBuilder.forOutBound(srcHasNamespaceStr, "sNs", "sType") arangoQueryBuilder.forOutBound(srcHasNameStr, "sName", "sNs") } @@ -365,10 +350,7 @@ func (c *arangoClient) sourcesType(ctx context.Context, sourceSpec *model.Source values := map[string]any{} - arangoQueryBuilder := newForQuery(srcRootsStr, "sRoot") - arangoQueryBuilder.filter("sRoot", "root", "==", "@src") - values["src"] = "src" - arangoQueryBuilder.forOutBound(srcHasTypeStr, "sType", "sRoot") + arangoQueryBuilder := newForQuery(srcTypesStr, "sType") if sourceSpec.Type != nil { arangoQueryBuilder.filter("sType", "type", "==", "@srcType") values["srcType"] = *sourceSpec.Type @@ -413,10 +395,7 @@ func (c *arangoClient) sourcesType(ctx context.Context, sourceSpec *model.Source func (c *arangoClient) sourcesNamespace(ctx context.Context, sourceSpec *model.SourceSpec) ([]*model.Source, error) { values := map[string]any{} - arangoQueryBuilder := newForQuery(srcRootsStr, "sRoot") - arangoQueryBuilder.filter("sRoot", "root", "==", "@src") - values["src"] = "src" - arangoQueryBuilder.forOutBound(srcHasTypeStr, "sType", "sRoot") + arangoQueryBuilder := newForQuery(srcTypesStr, "sType") if sourceSpec.Type != nil { arangoQueryBuilder.filter("sType", "type", "==", "@srcType") values["srcType"] = *sourceSpec.Type diff --git a/pkg/assembler/clients/helpers/bulk.go b/pkg/assembler/clients/helpers/bulk.go index 284e38a0f0..aba42286aa 100644 --- a/pkg/assembler/clients/helpers/bulk.go +++ b/pkg/assembler/clients/helpers/bulk.go @@ -161,22 +161,16 @@ func GetBulkAssembler(ctx context.Context, gqlclient graphql.Client) func([]asse } } - // TODO(pxp928): add bulk ingestion for CertifyBad logger.Infof("assembling CertifyBad: %v", len(p.CertifyBad)) - for _, bad := range p.CertifyBad { - if err := ingestCertifyBad(ctx, gqlclient, bad); err != nil { - return fmt.Errorf("ingestCertifyBad failed with error: %w", err) + if err := ingestCertifyBads(ctx, gqlclient, p.CertifyBad); err != nil { + return fmt.Errorf("ingestCertifyBads failed with error: %w", err) - } } - // TODO(pxp928): add bulk ingestion for CertifyGood logger.Infof("assembling CertifyGood: %v", len(p.CertifyGood)) - for _, good := range p.CertifyGood { - if err := ingestCertifyGood(ctx, gqlclient, good); err != nil { - return fmt.Errorf("ingestCertifyGood failed with error: %w", err) + if err := ingestCertifyGoods(ctx, gqlclient, p.CertifyGood); err != nil { + return fmt.Errorf("ingestCertifyGoods failed with error: %w", err) - } } // TODO: add bulk ingestion for PointOfContact @@ -384,6 +378,118 @@ func ingestHasSBOMs(ctx context.Context, client graphql.Client, v []assembler.Ha return nil } +func ingestCertifyGoods(ctx context.Context, client graphql.Client, v []assembler.CertifyGoodIngest) error { + var pkgVersions []model.PkgInputSpec + var pkgNames []model.PkgInputSpec + var sources []model.SourceInputSpec + var artifacts []model.ArtifactInputSpec + var pkgVersionCertifyGoods []model.CertifyGoodInputSpec + var pkgNameCertifyGoods []model.CertifyGoodInputSpec + var srcCertifyGoods []model.CertifyGoodInputSpec + var artCertifyGoods []model.CertifyGoodInputSpec + for _, ingest := range v { + if err := validatePackageSourceOrArtifactInput(ingest.Pkg, ingest.Src, ingest.Artifact, "ingestCertifyGoods"); err != nil { + return fmt.Errorf("input validation failed for ingestCertifyGoods: %w", err) + } + if ingest.Pkg != nil { + if ingest.PkgMatchFlag.Pkg == model.PkgMatchTypeSpecificVersion { + pkgVersions = append(pkgVersions, *ingest.Pkg) + pkgVersionCertifyGoods = append(pkgVersionCertifyGoods, *ingest.CertifyGood) + } else { + pkgNames = append(pkgNames, *ingest.Pkg) + pkgNameCertifyGoods = append(pkgNameCertifyGoods, *ingest.CertifyGood) + } + } else if ingest.Src != nil { + sources = append(sources, *ingest.Src) + srcCertifyGoods = append(srcCertifyGoods, *ingest.CertifyGood) + } else { + artifacts = append(artifacts, *ingest.Artifact) + artCertifyGoods = append(artCertifyGoods, *ingest.CertifyGood) + } + } + if len(pkgVersions) > 0 { + _, err := model.CertifyGoodPkgs(ctx, client, pkgVersions, model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, pkgVersionCertifyGoods) + if err != nil { + return fmt.Errorf("CertifyGoodPkgs - specific version failed with error: %w", err) + } + } + if len(pkgNames) > 0 { + _, err := model.CertifyGoodPkgs(ctx, client, pkgNames, model.MatchFlags{Pkg: model.PkgMatchTypeAllVersions}, pkgNameCertifyGoods) + if err != nil { + return fmt.Errorf("CertifyGoodPkgs - all versions failed with error: %w", err) + } + } + if len(sources) > 0 { + _, err := model.CertifyGoodSrcs(ctx, client, sources, srcCertifyGoods) + if err != nil { + return fmt.Errorf("CertifyGoodSrcs failed with error: %w", err) + } + } + if len(artifacts) > 0 { + _, err := model.CertifyGoodArtifacts(ctx, client, artifacts, artCertifyGoods) + if err != nil { + return fmt.Errorf("CertifyGoodArtifacts failed with error: %w", err) + } + } + return nil +} + +func ingestCertifyBads(ctx context.Context, client graphql.Client, v []assembler.CertifyBadIngest) error { + var pkgVersions []model.PkgInputSpec + var pkgNames []model.PkgInputSpec + var sources []model.SourceInputSpec + var artifacts []model.ArtifactInputSpec + var pkgVersionCertifyBads []model.CertifyBadInputSpec + var pkgNameCertifyBads []model.CertifyBadInputSpec + var srcCertifyBads []model.CertifyBadInputSpec + var artCertifyBads []model.CertifyBadInputSpec + for _, ingest := range v { + if err := validatePackageSourceOrArtifactInput(ingest.Pkg, ingest.Src, ingest.Artifact, "ingestCertifyBads"); err != nil { + return fmt.Errorf("input validation failed for ingestCertifyBads: %w", err) + } + if ingest.Pkg != nil { + if ingest.PkgMatchFlag.Pkg == model.PkgMatchTypeSpecificVersion { + pkgVersions = append(pkgVersions, *ingest.Pkg) + pkgVersionCertifyBads = append(pkgVersionCertifyBads, *ingest.CertifyBad) + } else { + pkgNames = append(pkgNames, *ingest.Pkg) + pkgNameCertifyBads = append(pkgNameCertifyBads, *ingest.CertifyBad) + } + } else if ingest.Src != nil { + sources = append(sources, *ingest.Src) + srcCertifyBads = append(srcCertifyBads, *ingest.CertifyBad) + } else { + artifacts = append(artifacts, *ingest.Artifact) + artCertifyBads = append(artCertifyBads, *ingest.CertifyBad) + } + } + if len(pkgVersions) > 0 { + _, err := model.CertifyBadPkgs(ctx, client, pkgVersions, model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, pkgVersionCertifyBads) + if err != nil { + return fmt.Errorf("certifyBadPkgs - specific version failed with error: %w", err) + } + } + if len(pkgNames) > 0 { + _, err := model.CertifyBadPkgs(ctx, client, pkgNames, model.MatchFlags{Pkg: model.PkgMatchTypeAllVersions}, pkgNameCertifyBads) + if err != nil { + return fmt.Errorf("certifyBadPkgs - all versions failed with error: %w", err) + } + } + if len(sources) > 0 { + _, err := model.CertifyBadSrcs(ctx, client, sources, srcCertifyBads) + if err != nil { + return fmt.Errorf("CertifyBadSrcs failed with error: %w", err) + } + } + if len(artifacts) > 0 { + _, err := model.CertifyBadArtifacts(ctx, client, artifacts, artCertifyBads) + if err != nil { + return fmt.Errorf("CertifyBadArtifacts failed with error: %w", err) + } + } + return nil +} + func ingestIsOccurrences(ctx context.Context, client graphql.Client, v []assembler.IsOccurrenceIngest) error { var pkgs []model.PkgInputSpec var sources []model.SourceInputSpec