As 3174 seperate object key and db meta data (#102)

* Update search options

* Update search options

* Update to released version of nameindexer

* Fix tests return the entire cloudevent

e2e/setup_test.go

@@ -11,7 +11,6 @@ import (
 	"github.com/99designs/gqlgen/client"
 	"github.com/DIMO-Network/clickhouse-infra/pkg/container"
 	"github.com/DIMO-Network/model-garage/pkg/cloudevent"
-	"github.com/DIMO-Network/nameindexer"
 	"github.com/DIMO-Network/nameindexer/pkg/clickhouse/indexrepo"
 	"github.com/DIMO-Network/telemetry-api/internal/app"
 	"github.com/DIMO-Network/telemetry-api/internal/config"
@@ -97,17 +96,13 @@ func GetTestServices(t *testing.T) *TestServices {
 }
 
 func StoreSampleVC(ctx context.Context, idxSrv *indexrepo.Service, bucket string, testVC string) error {
-	hdr := cloudevent.CloudEventHeader{}
-	err := json.Unmarshal([]byte(testVC), &hdr)
+	event := cloudevent.CloudEvent[json.RawMessage]{}
+	err := json.Unmarshal([]byte(testVC), &event)
 	if err != nil {
 		return fmt.Errorf("failed to unmarshal VC: %w", err)
 	}
-	cloudIdx, err := nameindexer.CloudEventToCloudIndex(&hdr, nameindexer.DefaultSecondaryFiller)
-	if err != nil {
-		return fmt.Errorf("failed to convert VC to cloud index: %w", err)
-	}
 
-	err = idxSrv.StoreCloudEventObject(ctx, cloudIdx, bucket, []byte(testVC))
+	err = idxSrv.StoreCloudEvent(ctx, bucket, event)
 	if err != nil {
 		return fmt.Errorf("failed to store VC: %w", err)
 	}

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/DIMO-Network/attestation-api v0.0.11
 	github.com/DIMO-Network/clickhouse-infra v0.0.3
 	github.com/DIMO-Network/model-garage v0.4.2
-	github.com/DIMO-Network/nameindexer v0.0.11
+	github.com/DIMO-Network/nameindexer v0.0.12
 	github.com/DIMO-Network/shared v0.12.6
 	github.com/Khan/genqlient v0.7.0
 	github.com/auth0/go-jwt-middleware/v2 v2.2.1

go.sum

@@ -80,8 +80,8 @@ github.com/DIMO-Network/clickhouse-infra v0.0.3 h1:B6/4IY9IxLcyydET14IjHUT+A5SDE
 github.com/DIMO-Network/clickhouse-infra v0.0.3/go.mod h1:NtpQ1btkPzebDvpYYygeqiiBmJ/q5oJb/T/JWzUVRlk=
 github.com/DIMO-Network/model-garage v0.4.2 h1:TF+RNC1n1Y58QSdzmKaf5REixdariAP4tqHihzjSe70=
 github.com/DIMO-Network/model-garage v0.4.2/go.mod h1:SXaBHFFARooOEoaV1wFaOC1K0LMt0bovkILsFXoUQTg=
-github.com/DIMO-Network/nameindexer v0.0.11 h1:VmonlyW98BSIX7c4hwlRqpzIBLlyMby4x5WvzvANbCA=
-github.com/DIMO-Network/nameindexer v0.0.11/go.mod h1:Hd+vs7PCqSIdAdkcHprdqQjsMED5fuiHP6TNKdqUajs=
+github.com/DIMO-Network/nameindexer v0.0.12 h1:Xh+W27pRrepKCY/oZbp9iNpg311K0qInD1McWB1hkjs=
+github.com/DIMO-Network/nameindexer v0.0.12/go.mod h1:Hd+vs7PCqSIdAdkcHprdqQjsMED5fuiHP6TNKdqUajs=
 github.com/DIMO-Network/shared v0.12.6 h1:vAnhmYnk21hBjFaSY1bA+oGJZENCsK7OmtSwF4O1Zqg=
 github.com/DIMO-Network/shared v0.12.6/go.mod h1:dtoa/pN19FFR/ghTsUWJynQClwvc+cIlf2mPDaUBDsE=
 github.com/DIMO-Network/yaml v0.1.0 h1:KQ3oKHUZETchR6Pxbmmol3e4ewrPv/q8cEwqxfwyZbU=

internal/repositories/vc/vc.go

@@ -10,13 +10,14 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/DIMO-Network/attestation-api/pkg/verifiable"
 	"github.com/DIMO-Network/model-garage/pkg/cloudevent"
-	"github.com/DIMO-Network/nameindexer"
 	"github.com/DIMO-Network/nameindexer/pkg/clickhouse/indexrepo"
 	"github.com/DIMO-Network/telemetry-api/internal/graph/model"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/rs/zerolog"
 )
 
+var vcType = cloudevent.TypeVerifableCredential
+
 type Repository struct {
 	logger         *zerolog.Logger
 	indexService   *indexrepo.Service
@@ -42,41 +43,40 @@ func New(chConn clickhouse.Conn, objGetter indexrepo.ObjectGetter, vcBucketName,
 
 // GetLatestVINVC fetches the latest VIN VC for the given vehicle.
 func (r *Repository) GetLatestVINVC(ctx context.Context, vehicleTokenID uint32) (*model.Vinvc, error) {
-	filler := nameindexer.FillerVerifiableCredential
 	vehicleDID := cloudevent.NFTDID{
 		ChainID:         r.chainID,
 		ContractAddress: r.vehicleAddress,
 		TokenID:         vehicleTokenID,
 	}
-	opts := indexrepo.CloudEventSearchOptions{
-		DataType:      &r.vinVCDataType,
-		PrimaryFiller: &filler,
-		Subject:       &vehicleDID,
+	opts := &indexrepo.SearchOptions{
+		DataVersion: &r.vinVCDataType,
+		Type:        &vcType,
+		Subject:     &vehicleDID,
 	}
-	dataObj, err := r.indexService.GetLatestCloudEventData(ctx, r.vcBucket, opts)
+	dataObj, err := r.indexService.GetLatestCloudEvent(ctx, r.vcBucket, opts)
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) {
 			return nil, nil
 		}
 		r.logger.Error().Err(err).Msg("failed to get latest VIN VC data")
 		return nil, errors.New("internal error")
 	}
-	msg := cloudevent.CloudEvent[verifiable.Credential]{}
-	if err := json.Unmarshal(dataObj.Data, &msg); err != nil {
+	cred := verifiable.Credential{}
+	if err := json.Unmarshal(dataObj.Data, &cred); err != nil {
 		r.logger.Error().Err(err).Msg("failed to unmarshal VIN VC")
 		return nil, errors.New("internal error")
 	}
 
 	var expiresAt *time.Time
-	if expirationDate, err := time.Parse(time.RFC3339, msg.Data.ValidTo); err == nil {
+	if expirationDate, err := time.Parse(time.RFC3339, cred.ValidTo); err == nil {
 		expiresAt = &expirationDate
 	}
 	var createdAt *time.Time
-	if issuanceDate, err := time.Parse(time.RFC3339, msg.Data.ValidFrom); err == nil {
+	if issuanceDate, err := time.Parse(time.RFC3339, cred.ValidFrom); err == nil {
 		createdAt = &issuanceDate
 	}
 	credSubject := verifiable.VINSubject{}
-	if err := json.Unmarshal(msg.Data.CredentialSubject, &credSubject); err != nil {
+	if err := json.Unmarshal(cred.CredentialSubject, &credSubject); err != nil {
 		r.logger.Error().Err(err).Msg("failed to unmarshal VIN credential subject")
 		return nil, errors.New("internal error")
 	}
@@ -101,11 +101,15 @@ func (r *Repository) GetLatestVINVC(ctx context.Context, vehicleTokenID uint32)
 		vehicleContractAddress = &credSubject.VehicleContractAddress
 	}
 	tokenIDInt := int(credSubject.VehicleTokenID)
-
+	rawVC, err := json.Marshal(dataObj)
+	if err != nil {
+		r.logger.Error().Err(err).Msg("failed to marshal Raw VIN VC")
+		return nil, errors.New("internal error")
+	}
 	return &model.Vinvc{
 		ValidFrom:              createdAt,
 		ValidTo:                expiresAt,
-		RawVc:                  string(dataObj.Data),
+		RawVc:                  string(rawVC),
 		Vin:                    vin,
 		RecordedBy:             recordedBy,
 		RecordedAt:             recordedAt,
@@ -117,37 +121,36 @@ func (r *Repository) GetLatestVINVC(ctx context.Context, vehicleTokenID uint32)
 
 // GetLatestPOMVC fetches the latest POM VC for the given vehicle.
 func (r *Repository) GetLatestPOMVC(ctx context.Context, vehicleTokenID uint32) (*model.Pomvc, error) {
-	filler := nameindexer.FillerVerifiableCredential
 	vehicleDID := cloudevent.NFTDID{
 		ChainID:         r.chainID,
 		ContractAddress: r.vehicleAddress,
 		TokenID:         vehicleTokenID,
 	}
-	opts := indexrepo.CloudEventSearchOptions{
-		DataType:      &r.pomVCDataType,
-		PrimaryFiller: &filler,
-		Subject:       &vehicleDID,
+	opts := &indexrepo.SearchOptions{
+		DataVersion: &r.pomVCDataType,
+		Type:        &vcType,
+		Subject:     &vehicleDID,
 	}
-	dataObj, err := r.indexService.GetLatestCloudEventData(ctx, r.vcBucket, opts)
+	dataObj, err := r.indexService.GetLatestCloudEvent(ctx, r.vcBucket, opts)
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) {
 			return nil, nil
 		}
 		r.logger.Error().Err(err).Msg("failed to get latest POM VC data")
 		return nil, errors.New("internal error")
 	}
-	msg := cloudevent.CloudEvent[verifiable.Credential]{}
-	if err := json.Unmarshal(dataObj.Data, &msg); err != nil {
+	cred := verifiable.Credential{}
+	if err := json.Unmarshal(dataObj.Data, &cred); err != nil {
 		r.logger.Error().Err(err).Msg("failed to unmarshal POM VC")
 		return nil, errors.New("internal error")
 	}
 
 	var createdAt *time.Time
-	if issuanceDate, err := time.Parse(time.RFC3339, msg.Data.ValidFrom); err == nil {
+	if issuanceDate, err := time.Parse(time.RFC3339, cred.ValidFrom); err == nil {
 		createdAt = &issuanceDate
 	}
 	credSubject := verifiable.POMSubject{}
-	if err := json.Unmarshal(msg.Data.CredentialSubject, &credSubject); err != nil {
+	if err := json.Unmarshal(cred.CredentialSubject, &credSubject); err != nil {
 		r.logger.Error().Err(err).Msg("failed to unmarshal POM credential subject")
 		return nil, errors.New("internal error")
 	}

internal/repositories/vc/vc_test.go

@@ -50,7 +50,6 @@ func (m *MockRow) ScanStruct(any) error {
 }
 
 func TestGetLatestVC(t *testing.T) {
-
 	// Initialize variables
 	logger := zerolog.New(nil)
 	ctx := context.Background()