Feature/si 2713 make vin vcs available in telemetry

charts/telemetry-api/templates/secret.yaml

@@ -14,6 +14,12 @@ spec:
  - remoteRef:
  key: {{ .Release.Namespace }}/telemetry/clickhouse/pass
  secretKey: CLICKHOUSE_PASSWORD
+ - remoteRef:
+ key: {{ .Release.Namespace }}/telemetry/s3/access_key_id
+ secretKey: S3_AWS_ACCESS_KEY_ID
+ - remoteRef:
+ key: {{ .Release.Namespace }}/telemetry/s3/secret_access_key
+ secretKey: S3_AWS_SECRET_ACCESS_KEY
  secretStoreRef:
  kind: ClusterSecretStore
  name: aws-secretsmanager-secret-store

charts/telemetry-api/values.yaml

@@ -32,6 +32,10 @@ env:
  TOKEN_EXCHANGE_ISSUER_URL: https://auth-roles-rights.dev.dimo.zone
  VEHICLE_NFT_ADDRESS: '0x45fbCD3ef7361d156e8b16F5538AE36DEdf61Da8'
  MAX_REQUEST_DURATION: 5s
+ S3_AWS_REGION: us-east-2
+ VINVC_BUCKET: dimo-network-vinvc-dev
+ VINVC_DATA_TYPE: 'VINVCv0.0'
+
 service:
  type: ClusterIP
  ports:

cmd/telemetry-api/main.go

@@ -12,13 +12,18 @@ import (
  "github.com/99designs/gqlgen/graphql"
  "github.com/99designs/gqlgen/graphql/handler"
  "github.com/99designs/gqlgen/graphql/playground"
+ "github.com/DIMO-Network/clickhouse-infra/pkg/connect"
  "github.com/DIMO-Network/shared"
  "github.com/DIMO-Network/telemetry-api/internal/auth"
  "github.com/DIMO-Network/telemetry-api/internal/config"
  "github.com/DIMO-Network/telemetry-api/internal/graph"
  "github.com/DIMO-Network/telemetry-api/internal/limits"
  "github.com/DIMO-Network/telemetry-api/internal/repositories"
+ "github.com/DIMO-Network/telemetry-api/internal/repositories/vinvc"
  "github.com/DIMO-Network/telemetry-api/internal/service/ch"
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
  "github.com/gofiber/fiber/v2"
  "github.com/gofiber/fiber/v2/middleware/adaptor"
  "github.com/prometheus/client_golang/prometheus/promhttp"
@@ -45,8 +50,15 @@ func main() {
  logger.Fatal().Err(err).Msg("Couldn't create ClickHouse service.")
  }
  baseRepo := repositories.NewRepository(&repoLogger, chService)
-
- cfg := graph.Config{Resolvers: &graph.Resolver{Repository: baseRepo}}
+ vinvcRepo, err := newVinVCServiceFromSettings(settings, &logger)
+ if err != nil {
+ logger.Fatal().Err(err).Msg("Couldn't create VINVC repository.")
+ }
+ resolver := &graph.Resolver{
+ Repository: baseRepo,
+ VINVCRepo: vinvcRepo,
+ }
+ cfg := graph.Config{Resolvers: resolver}
  cfg.Directives.RequiresPrivilege = auth.RequiresPrivilegeCheck
  cfg.Directives.RequiresToken = auth.RequiresTokenCheck
  cfg.Directives.IsSignal = noOp
@@ -116,3 +128,32 @@ func errorHandler(log zerolog.Logger) func(ctx context.Context, e error) *gqlerr
 func noOp(ctx context.Context, obj interface{}, next graphql.Resolver) (res interface{}, err error) {
  return next(ctx)
 }
+
+func newVinVCServiceFromSettings(settings config.Settings, parentLogger *zerolog.Logger) (*vinvc.Repository, error) {
+ chConfig := settings.CLickhouse
+ chConfig.Database = settings.ClickhouseFileIndexDatabase
+ chConn, err := connect.GetClickhouseConn(&chConfig)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get clickhouse connection: %w", err)
+ }
+ s3Client, err := s3ClientFromSettings(&settings)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create s3 client: %w", err)
+ }
+ vinvcLogger := parentLogger.With().Str("component", "vinvc").Logger()
+ return vinvc.New(chConn, s3Client, settings.VINVCBucket, settings.VINVCDataType, &vinvcLogger), nil
+}
+
+// s3ClientFromSettings creates an S3 client from the given settings.
+func s3ClientFromSettings(settings *config.Settings) (*s3.Client, error) {
+ // Create an AWS session
+ conf := aws.Config{
+ Region: settings.S3AWSRegion,
+ Credentials: credentials.NewStaticCredentialsProvider(
+ settings.S3AWSAccessKeyID,
+ settings.S3AWSSecretAccessKey,
+ "",
+ ),
+ }
+ return s3.NewFromConfig(conf), nil
+}

go.mod

@@ -4,15 +4,20 @@ go 1.22.0
 
 require (
  github.com/99designs/gqlgen v0.17.49
- github.com/ClickHouse/clickhouse-go/v2 v2.23.2
- github.com/DIMO-Network/clickhouse-infra v0.0.0-20240625130842-c9eff0c8aabe
+ github.com/ClickHouse/clickhouse-go/v2 v2.26.0
+ github.com/DIMO-Network/attestation-api v0.0.2
+ github.com/DIMO-Network/clickhouse-infra v0.0.1
  github.com/DIMO-Network/model-garage v0.2.6
+ github.com/DIMO-Network/nameindexer v0.0.1
  github.com/DIMO-Network/shared v0.10.19
  github.com/auth0/go-jwt-middleware/v2 v2.2.1
- github.com/ethereum/go-ethereum v1.14.0
+ github.com/aws/aws-sdk-go-v2 v1.30.1
+ github.com/aws/aws-sdk-go-v2/credentials v1.17.23
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.57.1
+ github.com/ethereum/go-ethereum v1.14.6
  github.com/gofiber/fiber/v2 v2.52.5
- github.com/prometheus/client_golang v1.19.0
- github.com/rs/zerolog v1.32.0
+ github.com/prometheus/client_golang v1.19.1
+ github.com/rs/zerolog v1.33.0
  github.com/stretchr/testify v1.9.0
  github.com/vektah/gqlparser/v2 v2.5.16
  github.com/volatiletech/sqlboiler/v4 v4.16.2
@@ -23,23 +28,32 @@ require (
  dario.cat/mergo v1.0.0 // indirect
  github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
  github.com/ClickHouse/ch-go v0.61.5 // indirect
- github.com/Microsoft/go-winio v0.6.1 // indirect
+ github.com/Microsoft/go-winio v0.6.2 // indirect
  github.com/Microsoft/hcsshim v0.11.4 // indirect
  github.com/agnivade/levenshtein v1.1.1 // indirect
  github.com/andybalholm/brotli v1.1.0 // indirect
  github.com/avast/retry-go/v4 v4.6.0 // indirect
- github.com/aws/aws-sdk-go-v2 v1.30.1 // indirect
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
  github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect
  github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13 // indirect
  github.com/aws/aws-sdk-go-v2/service/kms v1.35.1 // indirect
  github.com/aws/smithy-go v1.20.3 // indirect
  github.com/beorn7/perks v1.0.1 // indirect
+ github.com/btcsuite/btcd/btcec/v2 v2.3.3 // indirect
+ github.com/btcsuite/btcd/btcutil v1.1.5 // indirect
+ github.com/btcsuite/btcutil v1.0.2 // indirect
  github.com/cenkalti/backoff/v4 v4.2.1 // indirect
  github.com/cespare/xxhash/v2 v2.3.0 // indirect
  github.com/containerd/containerd v1.7.15 // indirect
  github.com/containerd/log v0.1.0 // indirect
  github.com/cpuguy83/dockercfg v0.3.1 // indirect
  github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
  github.com/distribution/reference v0.5.0 // indirect
  github.com/docker/docker v26.1.1+incompatible // indirect
  github.com/docker/go-connections v0.5.0 // indirect
@@ -73,17 +87,20 @@ require (
  github.com/moby/sys/user v0.1.0 // indirect
  github.com/moby/term v0.5.0 // indirect
  github.com/morikuni/aec v1.0.0 // indirect
+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
  github.com/opencontainers/go-digest v1.0.0 // indirect
  github.com/opencontainers/image-spec v1.1.0 // indirect
  github.com/paulmach/orb v0.11.1 // indirect
  github.com/pierrec/lz4/v4 v4.1.21 // indirect
+ github.com/piprate/json-gold v0.5.0 // indirect
  github.com/pkg/errors v0.9.1 // indirect
  github.com/pmezard/go-difflib v1.0.0 // indirect
  github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
+ github.com/pquerna/cachecontrol v0.2.0 // indirect
  github.com/pressly/goose/v3 v3.20.0 // indirect
  github.com/prometheus/client_model v0.6.1 // indirect
- github.com/prometheus/common v0.53.0 // indirect
- github.com/prometheus/procfs v0.14.0 // indirect
+ github.com/prometheus/common v0.55.0 // indirect
+ github.com/prometheus/procfs v0.15.1 // indirect
  github.com/rivo/uniseg v0.4.7 // indirect
  github.com/segmentio/asm v1.2.0 // indirect
  github.com/sethvargo/go-retry v0.2.4 // indirect
@@ -107,21 +124,19 @@ require (
  github.com/volatiletech/strmangle v0.0.6 // indirect
  github.com/yusufpapurcu/wmi v1.2.4 // indirect
  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
- go.opentelemetry.io/otel v1.26.0 // indirect
- go.opentelemetry.io/otel/metric v1.26.0 // indirect
- go.opentelemetry.io/otel/trace v1.26.0 // indirect
+ go.opentelemetry.io/otel v1.27.0 // indirect
+ go.opentelemetry.io/otel/metric v1.27.0 // indirect
+ go.opentelemetry.io/otel/trace v1.27.0 // indirect
  go.uber.org/multierr v1.11.0 // indirect
  golang.org/x/crypto v0.24.0 // indirect
- golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
- golang.org/x/mod v0.18.0 // indirect
+ golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
  golang.org/x/net v0.26.0 // indirect
  golang.org/x/sync v0.7.0 // indirect
  golang.org/x/sys v0.21.0 // indirect
  golang.org/x/text v0.16.0 // indirect
- golang.org/x/tools v0.22.0 // indirect
  golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
  google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
- google.golang.org/grpc v1.64.0 // indirect
+ google.golang.org/grpc v1.65.0 // indirect
  google.golang.org/protobuf v1.34.2 // indirect
  gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect
  gopkg.in/yaml.v3 v3.0.1 // indirect