From 527c2da37dcdc97b3f5eb39a0f408a0a845b0b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28puerco=29?= Date: Mon, 9 Dec 2024 17:45:29 +0100 Subject: [PATCH] Wire struct data source to DS machinery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit connects the new structured data source to the data source machinery. Signed-off-by: Adolfo García Veytia (puerco) --- internal/datasources/factory.go | 3 +++ internal/datasources/service/convert.go | 25 ++++++++++++++++++++ internal/datasources/service/service.go | 17 +++++++++++++ pkg/api/protobuf/go/minder/v1/datasources.go | 2 ++ pkg/api/protobuf/go/minder/v1/validators.go | 9 +++++++ pkg/datasources/v1/datasources.go | 2 ++ 6 files changed, 58 insertions(+) diff --git a/internal/datasources/factory.go b/internal/datasources/factory.go index e38e30493e..67ca3f90f5 100644 --- a/internal/datasources/factory.go +++ b/internal/datasources/factory.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/mindersec/minder/internal/datasources/rest" + "github.com/mindersec/minder/internal/datasources/structured" minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1" v1datasources "github.com/mindersec/minder/pkg/datasources/v1" ) @@ -24,6 +25,8 @@ func BuildFromProtobuf(ds *minderv1.DataSource) (v1datasources.DataSource, error } switch ds.GetDriver().(type) { + case *minderv1.DataSource_Structured: + return structured.NewStructDataSource(ds.GetStructured()) case *minderv1.DataSource_Rest: return rest.NewRestDataSource(ds.GetRest()) default: diff --git a/internal/datasources/service/convert.go b/internal/datasources/service/convert.go index 5666b5056a..8c38629389 100644 --- a/internal/datasources/service/convert.go +++ b/internal/datasources/service/convert.go @@ -33,6 +33,8 @@ func dataSourceDBToProtobuf(ds db.DataSource, dsfuncs []db.DataSourcesFunction) dsfType := dsfuncs[0].Type switch dsfType { + case v1datasources.DataSourceDriverStruct: + return dataSourceStructDBToProtobuf(outds, dsfuncs) case v1datasources.DataSourceDriverRest: return dataSourceRestDBToProtobuf(outds, dsfuncs) default: @@ -60,3 +62,26 @@ func dataSourceRestDBToProtobuf(ds *minderv1.DataSource, dsfuncs []db.DataSource return ds, nil } + +func dataSourceStructDBToProtobuf(ds *minderv1.DataSource, dsfuncs []db.DataSourcesFunction) (*minderv1.DataSource, error) { + // At this point we have already validated that we have at least one function. + ds.Driver = &minderv1.DataSource_Structured{ + Structured: &minderv1.StructDataSource{ + Def: make(map[string]*minderv1.StructDataSource_Def, len(dsfuncs)), + }, + } + + for _, dsf := range dsfuncs { + key := dsf.Name + dsfToParse := &minderv1.StructDataSource_Def{ + Path: &minderv1.StructDataSource_Def_Path{}, + } + if err := protojson.Unmarshal(dsf.Definition, dsfToParse); err != nil { + return nil, fmt.Errorf("failed to unmarshal data source definition for %s: %w", key, err) + } + + ds.GetStructured().Def[key] = dsfToParse + } + + return ds, nil +} diff --git a/internal/datasources/service/service.go b/internal/datasources/service/service.go index fc32dc40fe..375f0eadc9 100644 --- a/internal/datasources/service/service.go +++ b/internal/datasources/service/service.go @@ -431,6 +431,23 @@ func addDataSourceFunctions( projectID uuid.UUID, ) error { switch drv := ds.GetDriver().(type) { + case *minderv1.DataSource_Structured: + for name, def := range drv.Structured.GetDef() { + defBytes, err := protojson.Marshal(def) + if err != nil { + return fmt.Errorf("failed to marshal structured data definition: %w", err) + } + + if _, err := tx.AddDataSourceFunction(ctx, db.AddDataSourceFunctionParams{ + DataSourceID: dsID, + ProjectID: projectID, + Name: name, + Type: v1datasources.DataSourceDriverStruct, + Definition: defBytes, + }); err != nil { + return fmt.Errorf("failed to create data source function: %w", err) + } + } case *minderv1.DataSource_Rest: for name, def := range drv.Rest.GetDef() { defBytes, err := protojson.Marshal(def) diff --git a/pkg/api/protobuf/go/minder/v1/datasources.go b/pkg/api/protobuf/go/minder/v1/datasources.go index 82560220f7..b494427fde 100644 --- a/pkg/api/protobuf/go/minder/v1/datasources.go +++ b/pkg/api/protobuf/go/minder/v1/datasources.go @@ -25,6 +25,8 @@ func (ds *DataSource) GetDriverType() string { switch ds.GetDriver().(type) { case *DataSource_Rest: return v1datasources.DataSourceDriverRest + case *DataSource_Structured: + return v1datasources.DataSourceDriverStruct default: return "unknown" } diff --git a/pkg/api/protobuf/go/minder/v1/validators.go b/pkg/api/protobuf/go/minder/v1/validators.go index 83b8675218..d21bc9fffb 100644 --- a/pkg/api/protobuf/go/minder/v1/validators.go +++ b/pkg/api/protobuf/go/minder/v1/validators.go @@ -549,6 +549,15 @@ func (ds *DataSource) Validate() error { return val.Validate() } +// Validate checks a structured data source to ensure it is valid +func (dsStructuredDriver *DataSource_Structured) Validate() error { + if dsStructuredDriver == nil || dsStructuredDriver.Structured == nil { + return fmt.Errorf("%w: structured driver is nil", ErrValidationFailed) + } + + return nil +} + // Validate is the entrypoint for the actual driver's validation func (dsRestDriver *DataSource_Rest) Validate() error { if dsRestDriver == nil { diff --git a/pkg/datasources/v1/datasources.go b/pkg/datasources/v1/datasources.go index 7e06699f99..c5bbc6c5cb 100644 --- a/pkg/datasources/v1/datasources.go +++ b/pkg/datasources/v1/datasources.go @@ -13,6 +13,8 @@ import ( //go:generate go run go.uber.org/mock/mockgen -package mock_$GOPACKAGE -destination=./mock/$GOFILE -source=./$GOFILE const ( + // DataSourceDriverStruct is the driver type for the structured data source + DataSourceDriverStruct = "structured" // DataSourceDriverRest is the driver type for a REST data source. DataSourceDriverRest = "rest" )