From 88d5bcde4dea9a1da8aacb3ff337dc3447fcca68 Mon Sep 17 00:00:00 2001 From: Kevin Joiner <10265309+KevinJoiner@users.noreply.github.com> Date: Wed, 20 Nov 2024 19:13:47 -0500 Subject: [PATCH 1/2] Add end 2 end testing --- .gitignore | 2 +- cmd/telemetry-api/main.go | 126 ++---------------------- e2e/auth_server_test.go | 161 +++++++++++++++++++++++++++++++ e2e/clickhouse_container_test.go | 56 +++++++++++ e2e/identity_api_test.go | 72 ++++++++++++++ e2e/s3_server_test.go | 101 +++++++++++++++++++ e2e/setup_test.go | 130 +++++++++++++++++++++++++ e2e/signals_latest_test.go | 151 +++++++++++++++++++++++++++++ go.mod | 13 ++- go.sum | 54 ++++++++++- internal/app/app.go | 146 ++++++++++++++++++++++++++++ 11 files changed, 886 insertions(+), 126 deletions(-) create mode 100644 e2e/auth_server_test.go create mode 100644 e2e/clickhouse_container_test.go create mode 100644 e2e/identity_api_test.go create mode 100644 e2e/s3_server_test.go create mode 100644 e2e/setup_test.go create mode 100644 e2e/signals_latest_test.go create mode 100644 internal/app/app.go diff --git a/.gitignore b/.gitignore index b13abab..b229777 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ # Test binary, built with `go test -c` *.test bin/ - +.DS_Store # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/cmd/telemetry-api/main.go b/cmd/telemetry-api/main.go index d92e791..a79ac05 100644 --- a/cmd/telemetry-api/main.go +++ b/cmd/telemetry-api/main.go @@ -1,108 +1,45 @@ package main import ( - "context" - "errors" "flag" "fmt" "net/http" "os" "strconv" - "github.com/99designs/gqlgen/graphql" - "github.com/99designs/gqlgen/graphql/handler" "github.com/99designs/gqlgen/graphql/playground" - "github.com/DIMO-Network/clickhouse-infra/pkg/connect" "github.com/DIMO-Network/shared" - "github.com/DIMO-Network/telemetry-api/internal/auth" + "github.com/DIMO-Network/telemetry-api/internal/app" "github.com/DIMO-Network/telemetry-api/internal/config" - "github.com/DIMO-Network/telemetry-api/internal/graph" - "github.com/DIMO-Network/telemetry-api/internal/limits" - "github.com/DIMO-Network/telemetry-api/internal/repositories" - "github.com/DIMO-Network/telemetry-api/internal/repositories/vc" - "github.com/DIMO-Network/telemetry-api/internal/service/ch" - "github.com/DIMO-Network/telemetry-api/internal/service/identity" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/ethereum/go-ethereum/common" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/adaptor" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog" - "github.com/vektah/gqlparser/v2/gqlerror" ) func main() { - ctx := context.Background() logger := zerolog.New(os.Stdout).With().Timestamp().Str("app", "telemetry-api").Logger() - // create a flag for the settings file + settingsFile := flag.String("settings", "settings.yaml", "settings file") flag.Parse() + settings, err := shared.LoadConfig[config.Settings](*settingsFile) if err != nil { logger.Fatal().Err(err).Msg("Couldn't load settings.") } - // create clickhouse connection - _ = ctx - idService := identity.NewService(settings.IdentityAPIURL, settings.IdentityAPIReqTimeoutSeconds) - repoLogger := logger.With().Str("component", "repository").Logger() - chService, err := ch.NewService(settings) + application, err := app.New(settings, &logger) if err != nil { - logger.Fatal().Err(err).Msg("Couldn't create ClickHouse service.") - } - baseRepo, err := repositories.NewRepository(&repoLogger, chService, settings.DeviceLastSeenBinHrs) - if err != nil { - logger.Fatal().Err(err).Msg("Couldn't create base repository.") - } - vcRepo, err := newVinVCServiceFromSettings(settings, &logger) - if err != nil { - logger.Fatal().Err(err).Msg("Couldn't create VINVC repository.") - } - resolver := &graph.Resolver{ - Repository: baseRepo, - IdentityService: idService, - VCRepo: vcRepo, + logger.Fatal().Err(err).Msg("Couldn't create application.") } - - cfg := graph.Config{Resolvers: resolver} - cfg.Directives.RequiresVehicleToken = auth.NewVehicleTokenCheck(settings.VehicleNFTAddress) - cfg.Directives.RequiresManufacturerToken = auth.NewManufacturerTokenCheck(settings.ManufacturerNFTAddress, idService) - cfg.Directives.RequiresPrivileges = auth.PrivilegeCheck - cfg.Directives.IsSignal = noOp - cfg.Directives.HasAggregation = noOp - cfg.Directives.OneOf = noOp + defer application.Cleanup() serveMonitoring(strconv.Itoa(settings.MonPort), &logger) - server := handler.NewDefaultServer(graph.NewExecutableSchema(cfg)) - errLogger := logger.With().Str("component", "gql").Logger() - server.SetErrorPresenter(errorHandler(errLogger)) - - logger.Info().Str("jwksUrl", settings.TokenExchangeJWTKeySetURL).Str("issuerURL", settings.TokenExchangeIssuer).Str("vehicleAddr", settings.VehicleNFTAddress).Msg("Privileges enabled.") - - authMiddleware, err := auth.NewJWTMiddleware(settings.TokenExchangeIssuer, settings.TokenExchangeJWTKeySetURL, settings.VehicleNFTAddress, settings.ManufacturerNFTAddress, &logger) - if err != nil { - logger.Fatal().Err(err).Msg("Couldn't create JWT middleware.") - } - - limiter, err := limits.New(settings.MaxRequestDuration) - if err != nil { - logger.Fatal().Err(err).Msg("Couldn't create request time limit middleware.") - } - http.Handle("/", playground.Handler("GraphQL playground", "/query")) - - authedHandler := limiter.AddRequestTimeout( - authMiddleware.CheckJWT( - auth.AddClaimHandler(server, &logger, settings.VehicleNFTAddress, settings.ManufacturerNFTAddress), - ), - ) - http.Handle("/query", authedHandler) + http.Handle("/query", application.Handler) logger.Info().Msgf("Server started on port: %d", settings.Port) - logger.Fatal().Err(http.ListenAndServe(fmt.Sprintf(":%d", settings.Port), nil)).Msg("Server shut down.") } @@ -122,52 +59,3 @@ func serveMonitoring(port string, logger *zerolog.Logger) *fiber.App { return monApp } - -// errorHandler is a custom error handler for gqlgen -func errorHandler(log zerolog.Logger) func(ctx context.Context, e error) *gqlerror.Error { - return func(ctx context.Context, e error) *gqlerror.Error { - var gqlErr *gqlerror.Error - if errors.As(e, &gqlErr) { - return gqlErr - } - log.Error().Err(e).Msg("Internal server error") - return gqlerror.Errorf("internal server error") - } -} - -func noOp(ctx context.Context, obj interface{}, next graphql.Resolver) (res interface{}, err error) { - return next(ctx) -} - -func newVinVCServiceFromSettings(settings config.Settings, parentLogger *zerolog.Logger) (*vc.Repository, error) { - chConfig := settings.CLickhouse - chConfig.Database = settings.ClickhouseFileIndexDatabase - chConn, err := connect.GetClickhouseConn(&chConfig) - if err != nil { - return nil, fmt.Errorf("failed to get clickhouse connection: %w", err) - } - s3Client, err := s3ClientFromSettings(&settings) - if err != nil { - return nil, fmt.Errorf("failed to create s3 client: %w", err) - } - vinvcLogger := parentLogger.With().Str("component", "vinvc").Logger() - if !common.IsHexAddress(settings.VehicleNFTAddress) { - return nil, fmt.Errorf("invalid vehicle address: %s", settings.VehicleNFTAddress) - } - vehicleAddr := common.HexToAddress(settings.VehicleNFTAddress) - return vc.New(chConn, s3Client, settings.VCBucket, settings.VINVCDataType, settings.POMVCDataType, uint64(settings.ChainID), vehicleAddr, &vinvcLogger), nil -} - -// s3ClientFromSettings creates an S3 client from the given settings. -func s3ClientFromSettings(settings *config.Settings) (*s3.Client, error) { - // Create an AWS session - conf := aws.Config{ - Region: settings.S3AWSRegion, - Credentials: credentials.NewStaticCredentialsProvider( - settings.S3AWSAccessKeyID, - settings.S3AWSSecretAccessKey, - "", - ), - } - return s3.NewFromConfig(conf), nil -} diff --git a/e2e/auth_server_test.go b/e2e/auth_server_test.go new file mode 100644 index 0000000..0328e2a --- /dev/null +++ b/e2e/auth_server_test.go @@ -0,0 +1,161 @@ +package e2e_test + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/go-jose/go-jose/v4" +) + +type mockAuthServer struct { + server *httptest.Server + signer jose.Signer + jwks jose.JSONWebKey + defaultClaims map[string]any +} + +func setupAuthServer(t *testing.T) *mockAuthServer { + t.Helper() + + // Generate RSA key + sk, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("Failed to generate RSA key: %v", err) + } + + // Generate key ID + b := make([]byte, 20) + if _, err := rand.Read(b); err != nil { + t.Fatalf("Failed to generate key ID: %v", err) + } + keyID := hex.EncodeToString(b) + + // Create JWK + jwk := jose.JSONWebKey{ + Key: sk.Public(), + KeyID: keyID, + Algorithm: string(jose.RS256), + Use: "sig", + } + + // Create signer + sig, err := jose.NewSigner(jose.SigningKey{ + Algorithm: jose.RS256, + Key: sk, + }, &jose.SignerOptions{ + ExtraHeaders: map[jose.HeaderKey]any{ + "kid": keyID, + }, + }) + if err != nil { + t.Fatalf("Failed to create signer: %v", err) + } + + defaultClaims := map[string]any{ + "aud": []string{ + "dimo.zone", + }, + "exp": 9722006230, + "iat": 1721833430, + "iss": "http://127.0.0.1:3003", + "sub": "0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF/39718", + } + + auth := &mockAuthServer{ + signer: sig, + jwks: jwk, + defaultClaims: defaultClaims, + } + + // Create test server with only JWKS endpoint + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/keys" { + http.NotFound(w, r) + return + } + json.NewEncoder(w).Encode(jose.JSONWebKeySet{ + Keys: []jose.JSONWebKey{jwk}, + }) + })) + + auth.server = server + return auth +} + +func (m *mockAuthServer) sign(claims map[string]interface{}) (string, error) { + b, err := json.Marshal(claims) + if err != nil { + return "", fmt.Errorf("failed to marshal claims: %w", err) + } + + out, err := m.signer.Sign(b) + if err != nil { + return "", fmt.Errorf("failed to sign claims: %w", err) + } + + token, err := out.CompactSerialize() + if err != nil { + return "", fmt.Errorf("failed to serialize token: %w", err) + } + + return token, nil +} + +func (m *mockAuthServer) CreateToken(t *testing.T, customClaims map[string]interface{}, privileges []int) string { + t.Helper() + + claims := make(map[string]interface{}) + for k, v := range m.defaultClaims { + claims[k] = v + } + for k, v := range customClaims { + claims[k] = v + } + + // Add privileges if provided + if privileges != nil { + claims["privilege_ids"] = privileges + } + + token, err := m.sign(claims) + if err != nil { + t.Fatalf("Failed to create token: %v", err) + } + + return token +} + +func (m *mockAuthServer) URL() string { + return m.server.URL +} + +func (m *mockAuthServer) Close() { + m.server.Close() +} + +// Helper function to create test tokens with specific claims and privileges +func (m *mockAuthServer) CreateVehicleToken(t *testing.T, tokenID string, privileges []int) string { + return m.CreateToken(t, map[string]interface{}{ + "token_id": tokenID, + "contract_address": "0x45fbCD3ef7361d156e8b16F5538AE36DEdf61Da8", + }, privileges) +} + +func (m *mockAuthServer) CreateManufacturerToken(t *testing.T, contractAddr string, tokenID string, privileges []int) string { + return m.CreateToken(t, map[string]interface{}{ + "contract_address": contractAddr, + "token_id": tokenID, + }, privileges) +} + +func (m *mockAuthServer) CreateUserToken(t *testing.T, userID string, privileges []int) string { + return m.CreateToken(t, map[string]interface{}{ + "sub": userID, + }, privileges) +} diff --git a/e2e/clickhouse_container_test.go b/e2e/clickhouse_container_test.go new file mode 100644 index 0000000..1f8b2ac --- /dev/null +++ b/e2e/clickhouse_container_test.go @@ -0,0 +1,56 @@ +package e2e_test + +import ( + "context" + "fmt" + "testing" + + chconfig "github.com/DIMO-Network/clickhouse-infra/pkg/connect/config" + "github.com/DIMO-Network/clickhouse-infra/pkg/container" + sigmigrations "github.com/DIMO-Network/model-garage/pkg/migrations" + "github.com/DIMO-Network/model-garage/pkg/vss" + "github.com/stretchr/testify/require" +) + +func setupClickhouseContainer(t *testing.T) *container.Container { + t.Helper() + ctx := context.Background() + + container, err := container.CreateClickHouseContainer(ctx, chconfig.Settings{}) + if err != nil { + t.Fatalf("Failed to create clickhouse container: %v", err) + } + + db, err := container.GetClickhouseAsDB() + if err != nil { + t.Fatalf("Failed to get clickhouse connection: %v", err) + } + + err = sigmigrations.RunGoose(ctx, []string{"up", "-v"}, db) + if err != nil { + t.Fatalf("Failed to run migrations: %v", err) + } + // err = indexmigrations.RunGoose(ctx, []string{"up", "-v"}, db) + // if err != nil { + // t.Fatalf("Failed to run migrations: %v", err) + // } + + return container +} + +// insertSignal inserts a test signal into Clickhouse +func insertSignal(t *testing.T, ch *container.Container, signals []vss.Signal) { + t.Helper() + + conn, err := ch.GetClickHouseAsConn() + require.NoError(t, err) + batch, err := conn.PrepareBatch(context.Background(), fmt.Sprintf("INSERT INTO %s", vss.TableName)) + require.NoError(t, err) + + for _, sig := range signals { + err := batch.AppendStruct(&sig) + require.NoError(t, err, "Failed to append signal to batch") + } + err = batch.Send() + require.NoError(t, err, "Failed to send batch") +} diff --git a/e2e/identity_api_test.go b/e2e/identity_api_test.go new file mode 100644 index 0000000..78a01a0 --- /dev/null +++ b/e2e/identity_api_test.go @@ -0,0 +1,72 @@ +package e2e_test + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "sync" +) + +type mockIdentityServer struct { + server *httptest.Server + responses map[string]interface{} // request payload hash -> response + mu sync.RWMutex +} + +func setupIdentityServer() *mockIdentityServer { + m := &mockIdentityServer{ + responses: make(map[string]interface{}), + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + // Read and hash the request body + body, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + // Get response for this exact request payload + m.mu.RLock() + response, exists := m.responses[string(body)] + m.mu.RUnlock() + + if !exists { + w.WriteHeader(http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + })) + + m.server = server + return m +} + +// SetRequestResponse sets a response for an exact request payload +func (m *mockIdentityServer) SetRequestResponse(request, response interface{}) error { + reqBytes, err := json.Marshal(request) + if err != nil { + return err + } + + m.mu.Lock() + defer m.mu.Unlock() + m.responses[string(reqBytes)] = response + return nil +} + +func (m *mockIdentityServer) URL() string { + return m.server.URL +} + +func (m *mockIdentityServer) Close() { + m.server.Close() +} diff --git a/e2e/s3_server_test.go b/e2e/s3_server_test.go new file mode 100644 index 0000000..c4c9c08 --- /dev/null +++ b/e2e/s3_server_test.go @@ -0,0 +1,101 @@ +package e2e_test + +import ( + "context" + "fmt" + "log" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/minio" +) + +type s3Server struct { + container testcontainers.Container + client *s3.Client + vcBucket string +} + +func setupS3Server(t *testing.T, vcBucket, pomBucket string) *s3Server { + t.Helper() + ctx := context.Background() + + minioContainer, err := minio.Run(ctx, "minio/minio:RELEASE.2024-01-16T16-07-38Z", + testcontainers.WithEnv(map[string]string{ + "MINIO_ACCESS_KEY": "minioadmin", + "MINIO_SECRET": "minioadmin", + }), + testcontainers.WithHostPortAccess(9000), + ) + + defer func() { + if err := testcontainers.TerminateContainer(minioContainer); err != nil { + log.Printf("failed to terminate container: %s", err) + } + }() + if err != nil { + require.NoError(t, err) + } + + mappedPort, err := minioContainer.MappedPort(ctx, "9000") + if err != nil { + t.Fatalf("Failed to get container port: %v", err) + } + + endpoint := fmt.Sprintf("http://localhost:%s", mappedPort.Port()) + + // Create S3 client + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: endpoint, + }, nil + }) + + cfg, err := config.LoadDefaultConfig(ctx, + config.WithEndpointResolverWithOptions(customResolver), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("minioadmin", "minioadmin", "")), + config.WithRegion("us-east-1"), + ) + if err != nil { + t.Fatalf("Failed to create AWS config: %v", err) + } + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true + }) + + s3srv := &s3Server{ + container: minioContainer, + client: client, + vcBucket: vcBucket, + } + + // Create VC bucket + _, err = client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: &vcBucket, + }) + if err != nil { + t.Fatalf("Failed to create VC bucket: %v", err) + } + return s3srv +} + +func (s *s3Server) GetClient() *s3.Client { + return s.client +} + +func (s *s3Server) GetVCBucket() string { + return s.vcBucket +} + +func (s *s3Server) Cleanup(t *testing.T) { + t.Helper() + if err := s.container.Terminate(context.Background()); err != nil { + t.Logf("Failed to terminate S3 container: %v", err) + } +} diff --git a/e2e/setup_test.go b/e2e/setup_test.go new file mode 100644 index 0000000..93d62ee --- /dev/null +++ b/e2e/setup_test.go @@ -0,0 +1,130 @@ +package e2e_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "sync" + "testing" + + "github.com/DIMO-Network/clickhouse-infra/pkg/container" + "github.com/DIMO-Network/model-garage/pkg/cloudevent" + "github.com/DIMO-Network/nameindexer" + "github.com/DIMO-Network/nameindexer/pkg/clickhouse/indexrepo" + "github.com/DIMO-Network/telemetry-api/internal/app" + "github.com/DIMO-Network/telemetry-api/internal/config" + "github.com/rs/zerolog" +) + +// TestServices holds all singleton service instances +type TestServices struct { + Identity *mockIdentityServer + Auth *mockAuthServer + IndexRepo *indexrepo.Service + CH *container.Container + Settings config.Settings +} + +var ( + testServices *TestServices + once sync.Once + cleanupOnce sync.Once + cleanup func() +) + +// GetTestServices returns singleton instances of all test services +func GetTestServices(t *testing.T) *TestServices { + t.Helper() + + once.Do(func() { + // Setup services + identity := setupIdentityServer() + auth := setupAuthServer(t) + // s3 := setupS3Server(t, "test-vc-bucket", "test-pom-bucket") + ch := setupClickhouseContainer(t) + // chConn, err := ch.GetClickHouseAsConn() + // require.NoError(t, err) + // indexService := indexrepo.New(chConn, s3.GetClient()) + // Create test settings + settings := config.Settings{ + Port: 8080, + MonPort: 9090, + IdentityAPIURL: identity.URL(), + IdentityAPIReqTimeoutSeconds: 5, + TokenExchangeJWTKeySetURL: auth.URL() + "/keys", + TokenExchangeIssuer: "http://127.0.0.1:3003", + VehicleNFTAddress: "0x45fbCD3ef7361d156e8b16F5538AE36DEdf61Da8", + ManufacturerNFTAddress: "0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF", + MaxRequestDuration: "1m", + S3AWSRegion: "us-east-1", + S3AWSAccessKeyID: "minioadmin", + S3AWSSecretAccessKey: "minioadmin", + // VCBucket: s3.GetVCBucket(), // Use bucket from S3 server + VINVCDataType: "vin", + POMVCDataType: "pom", + ChainID: 1, + CLickhouse: ch.Config(), + DeviceLastSeenBinHrs: 3, + } + // storeSampleVC(context.Background(), indexService, s3.GetVCBucket()) + testServices = &TestServices{ + Identity: identity, + Auth: auth, + // IndexRepo: indexService, + CH: ch, + Settings: settings, + } + + // Setup cleanup function + cleanup = func() { + cleanupOnce.Do(func() { + identity.Close() + auth.Close() + // s3.Cleanup(t) + ch.Terminate(context.Background()) + }) + } + }) + + // Register cleanup to run after all tests + t.Cleanup(func() { + cleanup() + }) + + return testServices + +} + +const testVC = `{"id":"2oYEQJDQFRYCI1UIf6QTy92MS3G","source":"0x0000000000000000000000000000000000000000","producer":"did:nft:137:0x9c94C395cBcBDe662235E0A9d3bB87Ad708561BA_31653","specversion":"1.0","subject":"did:nft:137:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF_39718","time":"2024-11-08T03:54:51.291563165Z","type":"dimo.verifiablecredential","datacontenttype":"application/json","dataversion":"VINVCv1.0","data":{"@context":["https://www.w3.org/ns/credentials/v2",{"vehicleIdentificationNumber":"https://schema.org/vehicleIdentificationNumber"},"https://attestation-api.dimo.zone/v1/vc/context"],"id":"urn:uuid:0b74cd3b-5998-4436-ada3-22dd6cfe2b3c","type":["VerifiableCredential","Vehicle"],"issuer":"https://attestation-api.dimo.zone/v1/vc/keys","validFrom":"2024-11-08T03:54:51Z","validTo":"2024-11-10T00:00:00Z","credentialSubject":{"id":"did:nft:137_erc721:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF_39718","vehicleTokenId":39718,"vehicleContractAddress":"eth:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF","vehicleIdentificationNumber":"3eMCZ5AN5PM647548","recordedBy":"did:nft:137:0x9c94C395cBcBDe662235E0A9d3bB87Ad708561BA_31653","recordedAt":"2024-11-08T00:40:29Z"},"credentialStatus":{"id":"https://attestation-api.dimo.zone/v1/vc/status/39718","type":"BitstringStatusListEntry","statusPurpose":"revocation","statusListIndex":0,"statusListCredential":"https://attestation-api.dimo.zone/v1/vc/status"},"proof":{"type":"DataIntegrityProof","cryptosuite":"ecdsa-rdfc-2019","verificationMethod":"https://attestation-api.dimo.zone/v1/vc/keys#key1","created":"2024-11-08T03:54:51Z","proofPurpose":"assertionMethod","proofValue":"381yXZFRShe2rrr9A3VFGeXS9izouz7Gor1GTb6Mwjkpge8eEn814QivzssEogoLKrzGN6WPKWBQrFLgfcsUhuaAWhS421Dn"}}}` + +func storeSampleVC(ctx context.Context, idxSrv *indexrepo.Service, bucket string) error { + hdr := cloudevent.CloudEventHeader{} + json.Unmarshal([]byte(testVC), &hdr) + cloudIdx, err := nameindexer.CloudEventToCloudIndex(&hdr, nameindexer.DefaultSecondaryFiller) + if err != nil { + return fmt.Errorf("failed to convert VC to cloud index: %w", err) + } + + err = idxSrv.StoreCloudEventObject(ctx, cloudIdx, bucket, []byte(testVC)) + if err != nil { + return fmt.Errorf("failed to store VC: %w", err) + } + return nil +} + +func NewGraphQLServer(t *testing.T, settings config.Settings) *httptest.Server { + t.Helper() + + logger := zerolog.New(os.Stdout).With().Timestamp().Logger() + + application, err := app.New(settings, &logger) + if err != nil { + t.Fatalf("Failed to create application: %v", err) + } + + t.Cleanup(application.Cleanup) + + return httptest.NewServer(application.Handler) +} diff --git a/e2e/signals_latest_test.go b/e2e/signals_latest_test.go new file mode 100644 index 0000000..b873f4e --- /dev/null +++ b/e2e/signals_latest_test.go @@ -0,0 +1,151 @@ +package e2e_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/DIMO-Network/model-garage/pkg/vss" + "github.com/DIMO-Network/telemetry-api/internal/service/ch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSignalsLatest(t *testing.T) { + services := GetTestServices(t) + smartCarTime := time.Date(2024, 11, 20, 22, 28, 17, 0, time.UTC) + autopiTime := time.Date(2024, 11, 1, 20, 1, 29, 0, time.UTC) + macaronTime := time.Date(2024, 3, 20, 20, 13, 57, 0, time.UTC) + // Set up test data in Clickhouse + signals := []vss.Signal{ + { + Source: ch.SourceTranslations["smartcar"], + Timestamp: smartCarTime, + Name: vss.FieldSpeed, + ValueNumber: 65.5, + TokenID: 39718, + }, + { + Source: ch.SourceTranslations["autopi"], + Timestamp: autopiTime, + Name: vss.FieldSpeed, + ValueNumber: 14, + TokenID: 39718, + }, + { + Source: ch.SourceTranslations["macaron"], + Timestamp: macaronTime, + Name: vss.FieldSpeed, + ValueNumber: 3, + TokenID: 39718, + }, + } + + insertSignal(t, services.CH, signals) + + // Create and set up GraphQL server + server := NewGraphQLServer(t, services.Settings) + defer server.Close() + + // Create auth token for vehicle + token := services.Auth.CreateVehicleToken(t, "39718", []int{1}) // Assuming 1 is the privilege needed + + // Execute the query + query := ` + query Latest_all { + smartcar: signalsLatest(filter: {source: "smartcar"}, tokenId: 39718) { + lastSeen + speed { + timestamp + value + } + } + autopi: signalsLatest(filter: {source: "autopi"}, tokenId: 39718) { + lastSeen + speed { + timestamp + value + } + } + macaron: signalsLatest(filter: {source: "macaron"}, tokenId: 39718) { + lastSeen + speed { + timestamp + value + } + } + tesla: signalsLatest(filter: {source: "tesla"}, tokenId: 39718) { + lastSeen + } + }` + + // Create request + body, err := json.Marshal(map[string]interface{}{ + "query": query, + }) + require.NoError(t, err) + + req, err := http.NewRequest("POST", server.URL, bytes.NewBuffer(body)) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + + // Execute request + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Parse response + var result struct { + Data struct { + Smartcar struct { + LastSeen string `json:"lastSeen"` + Speed *SignalWithTime `json:"speed"` + } `json:"smartcar"` + Autopi struct { + LastSeen string `json:"lastSeen"` + Speed *SignalWithTime `json:"speed"` + } `json:"autopi"` + Macaron struct { + LastSeen string `json:"lastSeen"` + Speed *SignalWithTime `json:"speed"` + } `json:"macaron"` + Tesla struct { + LastSeen *string `json:"lastSeen"` + } `json:"tesla"` + } `json:"data"` + } + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + fmt.Println(string(data)) + err = json.Unmarshal(data, &result) + require.NoError(t, err) + + // Assert the results + assert.Equal(t, smartCarTime.Format(time.RFC3339), result.Data.Smartcar.LastSeen) + assert.Equal(t, smartCarTime.Format(time.RFC3339), result.Data.Smartcar.Speed.Timestamp) + require.NotNil(t, result.Data.Autopi.Speed) + + assert.Equal(t, autopiTime.Format(time.RFC3339), result.Data.Autopi.LastSeen) + require.NotNil(t, result.Data.Autopi.Speed) + assert.Equal(t, autopiTime.Format(time.RFC3339), result.Data.Autopi.Speed.Timestamp) + assert.Equal(t, float64(14), result.Data.Autopi.Speed.Value) + + assert.Equal(t, macaronTime.Format(time.RFC3339), result.Data.Macaron.LastSeen) + require.NotNil(t, result.Data.Macaron.Speed) + assert.Equal(t, macaronTime.Format(time.RFC3339), result.Data.Macaron.Speed.Timestamp) + assert.Equal(t, float64(3), result.Data.Macaron.Speed.Value) + + assert.Nil(t, result.Data.Tesla.LastSeen) +} + +type SignalWithTime struct { + Timestamp string `json:"timestamp"` + Value float64 `json:"value"` +} diff --git a/go.mod b/go.mod index 2958866..a670b5a 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,17 @@ require ( github.com/Khan/genqlient v0.7.0 github.com/auth0/go-jwt-middleware/v2 v2.2.1 github.com/aws/aws-sdk-go-v2 v1.32.2 + github.com/aws/aws-sdk-go-v2/config v1.18.45 github.com/aws/aws-sdk-go-v2/credentials v1.17.41 github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 github.com/ethereum/go-ethereum v1.14.11 + github.com/go-jose/go-jose/v4 v4.0.4 github.com/gofiber/fiber/v2 v2.52.5 github.com/prometheus/client_golang v1.20.5 github.com/rs/zerolog v1.33.0 github.com/stretchr/testify v1.9.0 + github.com/testcontainers/testcontainers-go v0.34.0 + github.com/testcontainers/testcontainers-go/modules/minio v0.34.0 github.com/vektah/gqlparser/v2 v2.5.19 github.com/volatiletech/sqlboiler/v4 v4.16.2 go.uber.org/mock v0.5.0 @@ -35,14 +39,19 @@ require ( github.com/andybalholm/brotli v1.1.1 // indirect github.com/avast/retry-go/v4 v4.6.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect github.com/aws/aws-sdk-go-v2/service/kms v1.35.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect github.com/aws/smithy-go v1.22.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect @@ -52,7 +61,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect - github.com/cpuguy83/dockercfg v0.3.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/distribution/reference v0.6.0 // indirect @@ -74,6 +83,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/holiman/uint256 v1.3.1 // indirect github.com/jarcoal/httpmock v1.3.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -112,7 +122,6 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/sosodev/duration v1.3.1 // indirect github.com/spf13/cast v1.7.0 // indirect - github.com/testcontainers/testcontainers-go v0.33.0 // indirect github.com/testcontainers/testcontainers-go/modules/clickhouse v0.33.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index 66b0ee5..e42c8d5 100644 --- a/go.sum +++ b/go.sum @@ -124,22 +124,34 @@ github.com/auth0/go-jwt-middleware/v2 v2.2.1 h1:pqxEIwlCztD0T9ZygGfOrw4NK/F9iotn github.com/auth0/go-jwt-middleware/v2 v2.2.1/go.mod h1:CSi0tuu0QrALbWdiQZwqFL8SbBhj4e2MJzkvNfjY0Us= github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= +github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= +github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= @@ -148,6 +160,16 @@ github.com/aws/aws-sdk-go-v2/service/kms v1.35.3 h1:UPTdlTOwWUX49fVi7cymEN6hDqCw github.com/aws/aws-sdk-go-v2/service/kms v1.35.3/go.mod h1:gjDP16zn+WWalyaUqwCCioQ8gU8lzttCCc9jYsiQI/8= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 h1:xA6XhTF7PE89BCNHJbQi8VvPzcgMtmGC5dr8S8N7lHk= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -214,8 +236,8 @@ github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7np github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= -github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -280,6 +302,8 @@ github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7F github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= +github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -456,12 +480,17 @@ github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInw github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -475,6 +504,8 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -526,6 +557,12 @@ github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGA github.com/microsoft/go-mssqldb v0.17.0/go.mod h1:OkoNGhGEs8EZqchVTtochlXruEhEOaO4S0d2sB5aeGQ= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.68 h1:hTqSIfLlpXaKuNy4baAp4Jjy2sqZEN9hRxD0M4aOfrQ= +github.com/minio/minio-go/v7 v7.0.68/go.mod h1:XAvOPJQ5Xlzk5o3o/ArO2NMbhSGkimC+bpW/ngRKDmQ= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -550,9 +587,11 @@ github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcY github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= @@ -645,6 +684,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= @@ -691,6 +731,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -706,10 +748,12 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= -github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= +github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= +github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= github.com/testcontainers/testcontainers-go/modules/clickhouse v0.33.0 h1:YbB5DBkpgY+GlGPFqTSV1hzWPm3ZHirEyooZrj+ZXK0= github.com/testcontainers/testcontainers-go/modules/clickhouse v0.33.0/go.mod h1:qJuMPl9yWIWasmdBILM2uDk1Ny1kdeigcKMJ6A8PZz0= +github.com/testcontainers/testcontainers-go/modules/minio v0.34.0 h1:OpUqT7VV/d+wriDMHcCZCUfOoFE6wiHnGVzJOXqq8lU= +github.com/testcontainers/testcontainers-go/modules/minio v0.34.0/go.mod h1:0iaOtVNCzu04KcXHgmdNE7aelKaMUwC9x1M0oe6h1sw= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -1320,6 +1364,7 @@ gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKK gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1329,6 +1374,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..621be3a --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,146 @@ +package app + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/99designs/gqlgen/graphql" + "github.com/99designs/gqlgen/graphql/handler" + "github.com/DIMO-Network/clickhouse-infra/pkg/connect" + "github.com/DIMO-Network/telemetry-api/internal/auth" + "github.com/DIMO-Network/telemetry-api/internal/config" + "github.com/DIMO-Network/telemetry-api/internal/graph" + "github.com/DIMO-Network/telemetry-api/internal/limits" + "github.com/DIMO-Network/telemetry-api/internal/repositories" + "github.com/DIMO-Network/telemetry-api/internal/repositories/vc" + "github.com/DIMO-Network/telemetry-api/internal/service/ch" + "github.com/DIMO-Network/telemetry-api/internal/service/identity" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" + "github.com/vektah/gqlparser/v2/gqlerror" +) + +// App is the main application for the telemetry API. +type App struct { + Handler http.Handler + cleanup func() +} + +// New creates a new application. +func New(settings config.Settings, logger *zerolog.Logger) (*App, error) { + idService := identity.NewService(settings.IdentityAPIURL, settings.IdentityAPIReqTimeoutSeconds) + repoLogger := logger.With().Str("component", "repository").Logger() + chService, err := ch.NewService(settings) + if err != nil { + logger.Fatal().Err(err).Msg("Couldn't create ClickHouse service.") + } + baseRepo, err := repositories.NewRepository(&repoLogger, chService, settings.DeviceLastSeenBinHrs) + if err != nil { + logger.Fatal().Err(err).Msg("Couldn't create base repository.") + } + vcRepo, err := newVinVCServiceFromSettings(settings, logger) + if err != nil { + logger.Fatal().Err(err).Msg("Couldn't create VINVC repository.") + } + resolver := &graph.Resolver{ + Repository: baseRepo, + IdentityService: idService, + VCRepo: vcRepo, + } + + cfg := graph.Config{Resolvers: resolver} + cfg.Directives.RequiresVehicleToken = auth.NewVehicleTokenCheck(settings.VehicleNFTAddress) + cfg.Directives.RequiresManufacturerToken = auth.NewManufacturerTokenCheck(settings.ManufacturerNFTAddress, idService) + cfg.Directives.RequiresPrivileges = auth.PrivilegeCheck + cfg.Directives.IsSignal = noOp + cfg.Directives.HasAggregation = noOp + cfg.Directives.OneOf = noOp + + server := handler.NewDefaultServer(graph.NewExecutableSchema(cfg)) + errLogger := logger.With().Str("component", "gql").Logger() + server.SetErrorPresenter(errorHandler(errLogger)) + + logger.Info().Str("jwksUrl", settings.TokenExchangeJWTKeySetURL).Str("issuerURL", settings.TokenExchangeIssuer).Str("vehicleAddr", settings.VehicleNFTAddress).Msg("Privileges enabled.") + + authMiddleware, err := auth.NewJWTMiddleware(settings.TokenExchangeIssuer, settings.TokenExchangeJWTKeySetURL, settings.VehicleNFTAddress, settings.ManufacturerNFTAddress, logger) + if err != nil { + logger.Fatal().Err(err).Msg("Couldn't create JWT middleware.") + } + + limiter, err := limits.New(settings.MaxRequestDuration) + if err != nil { + logger.Fatal().Err(err).Msg("Couldn't create request time limit middleware.") + } + + authedHandler := limiter.AddRequestTimeout( + authMiddleware.CheckJWT( + auth.AddClaimHandler(server, logger, settings.VehicleNFTAddress, settings.ManufacturerNFTAddress), + ), + ) + + return &App{ + Handler: authedHandler, + cleanup: func() { + // TODO add cleanup logic for closing connections + }, + }, nil +} + +func (a *App) Cleanup() { + if a.cleanup != nil { + a.cleanup() + } +} + +func noOp(ctx context.Context, obj interface{}, next graphql.Resolver) (res interface{}, err error) { + return next(ctx) +} + +func newVinVCServiceFromSettings(settings config.Settings, parentLogger *zerolog.Logger) (*vc.Repository, error) { + chConfig := settings.CLickhouse + chConfig.Database = settings.ClickhouseFileIndexDatabase + chConn, err := connect.GetClickhouseConn(&chConfig) + if err != nil { + return nil, fmt.Errorf("failed to get clickhouse connection: %w", err) + } + s3Client, err := s3ClientFromSettings(&settings) + if err != nil { + return nil, fmt.Errorf("failed to create s3 client: %w", err) + } + vinvcLogger := parentLogger.With().Str("component", "vinvc").Logger() + if !common.IsHexAddress(settings.VehicleNFTAddress) { + return nil, fmt.Errorf("invalid vehicle address: %s", settings.VehicleNFTAddress) + } + vehicleAddr := common.HexToAddress(settings.VehicleNFTAddress) + return vc.New(chConn, s3Client, settings.VCBucket, settings.VINVCDataType, settings.POMVCDataType, uint64(settings.ChainID), vehicleAddr, &vinvcLogger), nil +} + +// s3ClientFromSettings creates an S3 client from the given settings. +func s3ClientFromSettings(settings *config.Settings) (*s3.Client, error) { + // Create an AWS session + conf := aws.Config{ + Region: settings.S3AWSRegion, + Credentials: credentials.NewStaticCredentialsProvider( + settings.S3AWSAccessKeyID, + settings.S3AWSSecretAccessKey, + "", + ), + } + return s3.NewFromConfig(conf), nil +} + +func errorHandler(log zerolog.Logger) func(ctx context.Context, e error) *gqlerror.Error { + return func(ctx context.Context, e error) *gqlerror.Error { + var gqlErr *gqlerror.Error + if errors.As(e, &gqlErr) { + return gqlErr + } + log.Error().Err(e).Msg("Internal server error") + return gqlerror.Errorf("internal server error") + } +} From e352eef28d394b38ccc1584683fc5f4bf2dae33f Mon Sep 17 00:00:00 2001 From: Kevin Joiner <10265309+KevinJoiner@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:39:05 -0500 Subject: [PATCH 2/2] Add vin VC test --- Makefile | 2 +- e2e/auth_server_test.go | 35 ++++++----- e2e/clickhouse_container_test.go | 27 ++++++--- e2e/identity_api_test.go | 9 ++- e2e/s3_server_test.go | 52 +++++++--------- e2e/setup_test.go | 82 ++++++++++++------------- e2e/vc_test.go | 101 +++++++++++++++++++++++++++++++ internal/app/app.go | 4 +- internal/config/settings.go | 1 + 9 files changed, 215 insertions(+), 98 deletions(-) create mode 100644 e2e/vc_test.go diff --git a/Makefile b/Makefile index 0c8da7d..d5cb13d 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ VERSION := $(shell git describe --tags || echo "v0.0.0") VER_CUT := $(shell echo $(VERSION) | cut -c2-) # Dependency versions -GOLANGCI_VERSION = v1.56.2 +GOLANGCI_VERSION = latest GQLGEN_VERSION = $(shell go list -m -f '{{.Version}}' github.com/99designs/gqlgen) MODEL_GARAGE_VERSION = $(shell go list -m -f '{{.Version}}' github.com/DIMO-Network/model-garage) MOCKGEN_VERSION = $(shell go list -m -f '{{.Version}}' go.uber.org/mock) diff --git a/e2e/auth_server_test.go b/e2e/auth_server_test.go index 0328e2a..5bffb8b 100644 --- a/e2e/auth_server_test.go +++ b/e2e/auth_server_test.go @@ -14,13 +14,15 @@ import ( ) type mockAuthServer struct { - server *httptest.Server - signer jose.Signer - jwks jose.JSONWebKey - defaultClaims map[string]any + server *httptest.Server + signer jose.Signer + jwks jose.JSONWebKey + defaultClaims map[string]any + VehicleContractAddress string + ManufacturerContractAddress string } -func setupAuthServer(t *testing.T) *mockAuthServer { +func setupAuthServer(t *testing.T, vehicleContractAddress, manufacturerContractAddress string) *mockAuthServer { t.Helper() // Generate RSA key @@ -68,9 +70,11 @@ func setupAuthServer(t *testing.T) *mockAuthServer { } auth := &mockAuthServer{ - signer: sig, - jwks: jwk, - defaultClaims: defaultClaims, + signer: sig, + jwks: jwk, + defaultClaims: defaultClaims, + VehicleContractAddress: vehicleContractAddress, + ManufacturerContractAddress: manufacturerContractAddress, } // Create test server with only JWKS endpoint @@ -79,9 +83,12 @@ func setupAuthServer(t *testing.T) *mockAuthServer { http.NotFound(w, r) return } - json.NewEncoder(w).Encode(jose.JSONWebKeySet{ + err := json.NewEncoder(w).Encode(jose.JSONWebKeySet{ Keys: []jose.JSONWebKey{jwk}, }) + if err != nil { + http.Error(w, "Failed to encode JWKS", http.StatusInternalServerError) + } })) auth.server = server @@ -117,7 +124,7 @@ func (m *mockAuthServer) CreateToken(t *testing.T, customClaims map[string]inter for k, v := range customClaims { claims[k] = v } - + // Add privileges if provided if privileges != nil { claims["privilege_ids"] = privileges @@ -143,14 +150,14 @@ func (m *mockAuthServer) Close() { func (m *mockAuthServer) CreateVehicleToken(t *testing.T, tokenID string, privileges []int) string { return m.CreateToken(t, map[string]interface{}{ "token_id": tokenID, - "contract_address": "0x45fbCD3ef7361d156e8b16F5538AE36DEdf61Da8", + "contract_address": m.VehicleContractAddress, }, privileges) } -func (m *mockAuthServer) CreateManufacturerToken(t *testing.T, contractAddr string, tokenID string, privileges []int) string { +func (m *mockAuthServer) CreateManufacturerToken(t *testing.T, tokenID string, privileges []int) string { return m.CreateToken(t, map[string]interface{}{ - "contract_address": contractAddr, - "token_id": tokenID, + "contract_address": m.ManufacturerContractAddress, + "token_id": tokenID, }, privileges) } diff --git a/e2e/clickhouse_container_test.go b/e2e/clickhouse_container_test.go index 1f8b2ac..d9b8cea 100644 --- a/e2e/clickhouse_container_test.go +++ b/e2e/clickhouse_container_test.go @@ -5,23 +5,25 @@ import ( "fmt" "testing" + "github.com/DIMO-Network/clickhouse-infra/pkg/connect" chconfig "github.com/DIMO-Network/clickhouse-infra/pkg/connect/config" "github.com/DIMO-Network/clickhouse-infra/pkg/container" sigmigrations "github.com/DIMO-Network/model-garage/pkg/migrations" "github.com/DIMO-Network/model-garage/pkg/vss" + indexmigrations "github.com/DIMO-Network/nameindexer/pkg/clickhouse/migrations" "github.com/stretchr/testify/require" ) -func setupClickhouseContainer(t *testing.T) *container.Container { +func setupClickhouseContainer(t *testing.T, indexDB string) *container.Container { t.Helper() ctx := context.Background() - container, err := container.CreateClickHouseContainer(ctx, chconfig.Settings{}) + chContainer, err := container.CreateClickHouseContainer(ctx, chconfig.Settings{}) if err != nil { t.Fatalf("Failed to create clickhouse container: %v", err) } - db, err := container.GetClickhouseAsDB() + db, err := chContainer.GetClickhouseAsDB() if err != nil { t.Fatalf("Failed to get clickhouse connection: %v", err) } @@ -30,12 +32,21 @@ func setupClickhouseContainer(t *testing.T) *container.Container { if err != nil { t.Fatalf("Failed to run migrations: %v", err) } - // err = indexmigrations.RunGoose(ctx, []string{"up", "-v"}, db) - // if err != nil { - // t.Fatalf("Failed to run migrations: %v", err) - // } + // Create index database + _, err = db.Exec("CREATE DATABASE IF NOT EXISTS " + indexDB) + require.NoError(t, err, "Failed to create index database") + chConfig := chContainer.Config() + chConfig.Database = indexDB + fileDB := connect.GetClickhouseDB(&chConfig) + if err != nil { + t.Fatalf("Failed to get clickhouse connection: %v", err) + } + err = indexmigrations.RunGoose(ctx, []string{"up", "-v"}, fileDB) + if err != nil { + t.Fatalf("Failed to run migrations: %v", err) + } - return container + return chContainer } // insertSignal inserts a test signal into Clickhouse diff --git a/e2e/identity_api_test.go b/e2e/identity_api_test.go index 78a01a0..c7c0729 100644 --- a/e2e/identity_api_test.go +++ b/e2e/identity_api_test.go @@ -43,7 +43,12 @@ func setupIdentityServer() *mockIdentityServer { } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) + + if err = json.NewEncoder(w).Encode(response); err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } })) m.server = server @@ -51,7 +56,7 @@ func setupIdentityServer() *mockIdentityServer { } // SetRequestResponse sets a response for an exact request payload -func (m *mockIdentityServer) SetRequestResponse(request, response interface{}) error { +func (m *mockIdentityServer) SetRequestResponse(request, response any) error { reqBytes, err := json.Marshal(request) if err != nil { return err diff --git a/e2e/s3_server_test.go b/e2e/s3_server_test.go index c4c9c08..dc05cfa 100644 --- a/e2e/s3_server_test.go +++ b/e2e/s3_server_test.go @@ -3,10 +3,8 @@ package e2e_test import ( "context" "fmt" - "log" "testing" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -15,13 +13,14 @@ import ( "github.com/testcontainers/testcontainers-go/modules/minio" ) -type s3Server struct { - container testcontainers.Container - client *s3.Client - vcBucket string +type mockS3Server struct { + container testcontainers.Container + client *s3.Client + vcBucket string + baseEndpoint *string } -func setupS3Server(t *testing.T, vcBucket, pomBucket string) *s3Server { +func setupS3Server(t *testing.T, vcBucket string) *mockS3Server { t.Helper() ctx := context.Background() @@ -33,11 +32,6 @@ func setupS3Server(t *testing.T, vcBucket, pomBucket string) *s3Server { testcontainers.WithHostPortAccess(9000), ) - defer func() { - if err := testcontainers.TerminateContainer(minioContainer); err != nil { - log.Printf("failed to terminate container: %s", err) - } - }() if err != nil { require.NoError(t, err) } @@ -46,33 +40,25 @@ func setupS3Server(t *testing.T, vcBucket, pomBucket string) *s3Server { if err != nil { t.Fatalf("Failed to get container port: %v", err) } - - endpoint := fmt.Sprintf("http://localhost:%s", mappedPort.Port()) + endpont := fmt.Sprintf("http://localhost:%s", mappedPort.Port()) // Create S3 client - customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { - return aws.Endpoint{ - URL: endpoint, - }, nil - }) - - cfg, err := config.LoadDefaultConfig(ctx, - config.WithEndpointResolverWithOptions(customResolver), + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("minioadmin", "minioadmin", "")), config.WithRegion("us-east-1"), ) if err != nil { t.Fatalf("Failed to create AWS config: %v", err) } - client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true + o.BaseEndpoint = &endpont }) - - s3srv := &s3Server{ - container: minioContainer, - client: client, - vcBucket: vcBucket, + s3srv := &mockS3Server{ + container: minioContainer, + client: client, + vcBucket: vcBucket, + baseEndpoint: &endpont, } // Create VC bucket @@ -85,15 +71,19 @@ func setupS3Server(t *testing.T, vcBucket, pomBucket string) *s3Server { return s3srv } -func (s *s3Server) GetClient() *s3.Client { +func (s *mockS3Server) GetClient() *s3.Client { return s.client } -func (s *s3Server) GetVCBucket() string { +func (s *mockS3Server) BaseEndpoint() *string { + return s.baseEndpoint +} + +func (s *mockS3Server) GetVCBucket() string { return s.vcBucket } -func (s *s3Server) Cleanup(t *testing.T) { +func (s *mockS3Server) Cleanup(t *testing.T) { t.Helper() if err := s.container.Terminate(context.Background()); err != nil { t.Logf("Failed to terminate S3 container: %v", err) diff --git a/e2e/setup_test.go b/e2e/setup_test.go index 93d62ee..3221462 100644 --- a/e2e/setup_test.go +++ b/e2e/setup_test.go @@ -18,13 +18,13 @@ import ( "github.com/rs/zerolog" ) -// TestServices holds all singleton service instances +// TestServices holds all singleton service instances. type TestServices struct { - Identity *mockIdentityServer - Auth *mockAuthServer - IndexRepo *indexrepo.Service - CH *container.Container - Settings config.Settings + Identity *mockIdentityServer + Auth *mockAuthServer + S3Server *mockS3Server + CH *container.Container + Settings config.Settings } var ( @@ -34,74 +34,74 @@ var ( cleanup func() ) +func TestMain(m *testing.M) { + os.Exit(m.Run()) + if cleanup != nil { + cleanup() + } +} + // GetTestServices returns singleton instances of all test services func GetTestServices(t *testing.T) *TestServices { t.Helper() once.Do(func() { - // Setup services - identity := setupIdentityServer() - auth := setupAuthServer(t) - // s3 := setupS3Server(t, "test-vc-bucket", "test-pom-bucket") - ch := setupClickhouseContainer(t) - // chConn, err := ch.GetClickHouseAsConn() - // require.NoError(t, err) - // indexService := indexrepo.New(chConn, s3.GetClient()) - // Create test settings settings := config.Settings{ Port: 8080, MonPort: 9090, - IdentityAPIURL: identity.URL(), IdentityAPIReqTimeoutSeconds: 5, - TokenExchangeJWTKeySetURL: auth.URL() + "/keys", TokenExchangeIssuer: "http://127.0.0.1:3003", - VehicleNFTAddress: "0x45fbCD3ef7361d156e8b16F5538AE36DEdf61Da8", - ManufacturerNFTAddress: "0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF", + VehicleNFTAddress: "0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF", + ManufacturerNFTAddress: "0x3b07e2A2ABdd0A9B8F7878bdE6487c502164B9dd", MaxRequestDuration: "1m", S3AWSRegion: "us-east-1", S3AWSAccessKeyID: "minioadmin", S3AWSSecretAccessKey: "minioadmin", - // VCBucket: s3.GetVCBucket(), // Use bucket from S3 server - VINVCDataType: "vin", - POMVCDataType: "pom", - ChainID: 1, - CLickhouse: ch.Config(), - DeviceLastSeenBinHrs: 3, + VCBucket: "test.vc.bucket", // TLDR keep the dots; If we don't use a non DNS resolved bucket name then the bucket lookup will attempt to use BUCKET_NAME.baseEndpoint + VINVCDataType: "VINVCv1.0", + POMVCDataType: "POMVCv1.0", + ChainID: 137, + ClickhouseFileIndexDatabase: "file_index", + DeviceLastSeenBinHrs: 3, } - // storeSampleVC(context.Background(), indexService, s3.GetVCBucket()) + + // Setup services + identity := setupIdentityServer() + auth := setupAuthServer(t, settings.VehicleNFTAddress, settings.ManufacturerNFTAddress) + s3 := setupS3Server(t, settings.VCBucket) + ch := setupClickhouseContainer(t, settings.ClickhouseFileIndexDatabase) + // Create test settings + + settings.CLickhouse = ch.Config() + settings.S3BaseEndpoint = s3.BaseEndpoint() + settings.IdentityAPIURL = identity.URL() + settings.TokenExchangeJWTKeySetURL = auth.URL() + "/keys" + testServices = &TestServices{ Identity: identity, Auth: auth, - // IndexRepo: indexService, + S3Server: s3, CH: ch, Settings: settings, } - - // Setup cleanup function cleanup = func() { cleanupOnce.Do(func() { identity.Close() auth.Close() - // s3.Cleanup(t) + s3.Cleanup(t) ch.Terminate(context.Background()) }) } }) - - // Register cleanup to run after all tests - t.Cleanup(func() { - cleanup() - }) - return testServices - } -const testVC = `{"id":"2oYEQJDQFRYCI1UIf6QTy92MS3G","source":"0x0000000000000000000000000000000000000000","producer":"did:nft:137:0x9c94C395cBcBDe662235E0A9d3bB87Ad708561BA_31653","specversion":"1.0","subject":"did:nft:137:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF_39718","time":"2024-11-08T03:54:51.291563165Z","type":"dimo.verifiablecredential","datacontenttype":"application/json","dataversion":"VINVCv1.0","data":{"@context":["https://www.w3.org/ns/credentials/v2",{"vehicleIdentificationNumber":"https://schema.org/vehicleIdentificationNumber"},"https://attestation-api.dimo.zone/v1/vc/context"],"id":"urn:uuid:0b74cd3b-5998-4436-ada3-22dd6cfe2b3c","type":["VerifiableCredential","Vehicle"],"issuer":"https://attestation-api.dimo.zone/v1/vc/keys","validFrom":"2024-11-08T03:54:51Z","validTo":"2024-11-10T00:00:00Z","credentialSubject":{"id":"did:nft:137_erc721:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF_39718","vehicleTokenId":39718,"vehicleContractAddress":"eth:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF","vehicleIdentificationNumber":"3eMCZ5AN5PM647548","recordedBy":"did:nft:137:0x9c94C395cBcBDe662235E0A9d3bB87Ad708561BA_31653","recordedAt":"2024-11-08T00:40:29Z"},"credentialStatus":{"id":"https://attestation-api.dimo.zone/v1/vc/status/39718","type":"BitstringStatusListEntry","statusPurpose":"revocation","statusListIndex":0,"statusListCredential":"https://attestation-api.dimo.zone/v1/vc/status"},"proof":{"type":"DataIntegrityProof","cryptosuite":"ecdsa-rdfc-2019","verificationMethod":"https://attestation-api.dimo.zone/v1/vc/keys#key1","created":"2024-11-08T03:54:51Z","proofPurpose":"assertionMethod","proofValue":"381yXZFRShe2rrr9A3VFGeXS9izouz7Gor1GTb6Mwjkpge8eEn814QivzssEogoLKrzGN6WPKWBQrFLgfcsUhuaAWhS421Dn"}}}` - -func storeSampleVC(ctx context.Context, idxSrv *indexrepo.Service, bucket string) error { +func StoreSampleVC(ctx context.Context, idxSrv *indexrepo.Service, bucket string, testVC string) error { hdr := cloudevent.CloudEventHeader{} - json.Unmarshal([]byte(testVC), &hdr) + err := json.Unmarshal([]byte(testVC), &hdr) + if err != nil { + return fmt.Errorf("failed to unmarshal VC: %w", err) + } cloudIdx, err := nameindexer.CloudEventToCloudIndex(&hdr, nameindexer.DefaultSecondaryFiller) if err != nil { return fmt.Errorf("failed to convert VC to cloud index: %w", err) diff --git a/e2e/vc_test.go b/e2e/vc_test.go new file mode 100644 index 0000000..6b95f5e --- /dev/null +++ b/e2e/vc_test.go @@ -0,0 +1,101 @@ +package e2e_test + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/DIMO-Network/clickhouse-infra/pkg/connect" + "github.com/DIMO-Network/nameindexer/pkg/clickhouse/indexrepo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVINVCLatest(t *testing.T) { + const testVC = `{"id":"2oYEQJDQFRYCI1UIf6QTy92MS3G","source":"0x0000000000000000000000000000000000000000","producer":"did:nft:137:0x9c94C395cBcBDe662235E0A9d3bB87Ad708561BA_31653","specversion":"1.0","subject":"did:nft:137:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF_39718","time":"2024-11-08T03:54:51.291563165Z","type":"dimo.verifiablecredential","datacontenttype":"application/json","dataversion":"VINVCv1.0","data":{"@context":["https://www.w3.org/ns/credentials/v2",{"vehicleIdentificationNumber":"https://schema.org/vehicleIdentificationNumber"},"https://attestation-api.dimo.zone/v1/vc/context"],"id":"urn:uuid:0b74cd3b-5998-4436-ada3-22dd6cfe2b3c","type":["VerifiableCredential","Vehicle"],"issuer":"https://attestation-api.dimo.zone/v1/vc/keys","validFrom":"2024-11-08T03:54:51Z","validTo":"2024-11-10T00:00:00Z","credentialSubject":{"id":"did:nft:137_erc721:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF_39718","vehicleTokenId":39718,"vehicleContractAddress":"eth:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF","vehicleIdentificationNumber":"3eMCZ5AN5PM647548","recordedBy":"did:nft:137:0x9c94C395cBcBDe662235E0A9d3bB87Ad708561BA_31653","recordedAt":"2024-11-08T00:40:29Z"},"credentialStatus":{"id":"https://attestation-api.dimo.zone/v1/vc/status/39718","type":"BitstringStatusListEntry","statusPurpose":"revocation","statusListIndex":0,"statusListCredential":"https://attestation-api.dimo.zone/v1/vc/status"},"proof":{"type":"DataIntegrityProof","cryptosuite":"ecdsa-rdfc-2019","verificationMethod":"https://attestation-api.dimo.zone/v1/vc/keys#key1","created":"2024-11-08T03:54:51Z","proofPurpose":"assertionMethod","proofValue":"381yXZFRShe2rrr9A3VFGeXS9izouz7Gor1GTb6Mwjkpge8eEn814QivzssEogoLKrzGN6WPKWBQrFLgfcsUhuaAWhS421Dn"}}}` + + services := GetTestServices(t) + + // Create and set up GraphQL server + server := NewGraphQLServer(t, services.Settings) + defer server.Close() + + cfg := services.CH.Config() + cfg.Database = services.Settings.ClickhouseFileIndexDatabase + chConn, err := connect.GetClickhouseConn(&cfg) + require.NoError(t, err) + indexService := indexrepo.New(chConn, services.S3Server.GetClient()) + if err = StoreSampleVC(context.Background(), indexService, services.S3Server.GetVCBucket(), testVC); err != nil { + t.Fatalf("Failed to store sample VC: %v", err) + } + + // Create auth token + token := services.Auth.CreateVehicleToken(t, "39718", []int{5}) + + query := ` + query VIN { + vinVCLatest(tokenId: 39718) { + vehicleTokenId + vin + recordedBy + recordedAt + countryCode + vehicleContractAddress + validFrom + validTo + rawVC + } + }` + + body, err := json.Marshal(map[string]interface{}{ + "query": query, + }) + require.NoError(t, err) + + req, err := http.NewRequest("POST", server.URL+"/query", bytes.NewBuffer(body)) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, http.StatusOK, resp.StatusCode) + + var result struct { + Data struct { + VINVC struct { + VehicleTokenID int `json:"vehicleTokenId"` + VIN string `json:"vin"` + RecordedBy string `json:"recordedBy"` + RecordedAt string `json:"recordedAt"` + CountryCode string `json:"countryCode"` + VehicleContractAddress string `json:"vehicleContractAddress"` + ValidFrom string `json:"validFrom"` + ValidTo string `json:"validTo"` + RawVC string `json:"rawVC"` + } `json:"vinVCLatest"` + } `json:"data"` + } + + err = json.NewDecoder(resp.Body).Decode(&result) + require.NoError(t, err) + + actual := result.Data.VINVC + expectedJSON, err := json.Marshal(testVC) + require.NoError(t, err) + actualJSON, err := json.Marshal(result.Data.VINVC.RawVC) + require.NoError(t, err) + + assert.JSONEq(t, string(expectedJSON), string(actualJSON)) + assert.Equal(t, 39718, actual.VehicleTokenID) + assert.Equal(t, "3eMCZ5AN5PM647548", actual.VIN) + assert.Equal(t, "did:nft:137:0x9c94C395cBcBDe662235E0A9d3bB87Ad708561BA_31653", actual.RecordedBy) + assert.Equal(t, "2024-11-08T00:40:29Z", actual.RecordedAt) + assert.Equal(t, "eth:0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF", actual.VehicleContractAddress) + assert.Equal(t, "2024-11-08T03:54:51Z", actual.ValidFrom) + assert.Equal(t, "2024-11-10T00:00:00Z", actual.ValidTo) +} diff --git a/internal/app/app.go b/internal/app/app.go index 621be3a..7059ebe 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -131,7 +131,9 @@ func s3ClientFromSettings(settings *config.Settings) (*s3.Client, error) { "", ), } - return s3.NewFromConfig(conf), nil + return s3.NewFromConfig(conf, func(o *s3.Options) { + o.BaseEndpoint = settings.S3BaseEndpoint + }), nil } func errorHandler(log zerolog.Logger) func(ctx context.Context, e error) *gqlerror.Error { diff --git a/internal/config/settings.go b/internal/config/settings.go index 63b72a0..934b994 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -15,6 +15,7 @@ type Settings struct { S3AWSRegion string `yaml:"S3_AWS_REGION"` S3AWSAccessKeyID string `yaml:"S3_AWS_ACCESS_KEY_ID"` S3AWSSecretAccessKey string `yaml:"S3_AWS_SECRET_ACCESS_KEY"` + S3BaseEndpoint *string `yaml:"S3_BASE_ENDPOINT"` VCBucket string `yaml:"VC_BUCKET"` VINVCDataType string `yaml:"VINVC_DATA_TYPE"` POMVCDataType string `yaml:"POMVC_DATA_TYPE"`