Skip to content

Commit

Permalink
Feature: saving blobs to s3 (#190)
Browse files Browse the repository at this point in the history 
* Feature: saving blobs to s3

* Remove debug

* Fix: comments

* Add receiving blob from R2

* Add blob receiving

* Fix: test

* DEBUG: comment timeout middleware
  • Loading branch information
@aopoltorzhicky
aopoltorzhicky authored Jun 3, 2024
1 parent ff4bbb1 commit 7f71895
Show file tree
Hide file tree
Showing 19 changed files with 1,019 additions and 80 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ adr:
@cp adr/adr-template.md adr/adr-$(NUM)-$(TITLE).md

generate:
go generate -v ./internal/storage ./internal/storage/types ./pkg/node ./internal/binance
go generate -v ./internal/blob ./internal/storage ./internal/storage/types ./pkg/node ./internal/binance

api-docs:
cd cmd/api && swag init --md markdown -parseDependency --parseInternal --parseDepth 1
Expand Down
36 changes: 28 additions & 8 deletions cmd/api/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package main

import (
"context"
"fmt"
"net/http"
"os"
"strconv"
Expand All @@ -17,9 +16,11 @@ import (
"github.com/celenium-io/celestia-indexer/cmd/api/gas"
"github.com/celenium-io/celestia-indexer/cmd/api/handler"
"github.com/celenium-io/celestia-indexer/cmd/api/handler/websocket"
"github.com/celenium-io/celestia-indexer/internal/blob"
"github.com/celenium-io/celestia-indexer/internal/profiler"
"github.com/celenium-io/celestia-indexer/internal/storage"
"github.com/celenium-io/celestia-indexer/internal/storage/postgres"
"github.com/celenium-io/celestia-indexer/pkg/node"
nodeApi "github.com/celenium-io/celestia-indexer/pkg/node/dal"
"github.com/celenium-io/celestia-indexer/pkg/node/rpc"
"github.com/dipdup-net/go-lib/config"
Expand Down Expand Up @@ -331,15 +332,11 @@ func initHandlers(ctx context.Context, e *echo.Echo, cfg Config, db postgres.Sto
}
}

datasource, ok := cfg.DataSources[cfg.ApiConfig.BlobReceiver]
if !ok {
panic(fmt.Sprintf("unknown data source pointed in blob_receiver: %s", cfg.ApiConfig.BlobReceiver))
blobReceiver, err := initBlobReceiver(ctx, cfg)
if err != nil {
panic(err)
}

blobReceiver := nodeApi.New(datasource.URL).
WithAuthToken(os.Getenv("CELESTIA_NODE_AUTH_TOKEN")).
WithRateLimit(datasource.RequestsPerSecond)

namespaceHandlers := handler.NewNamespaceHandler(db.Namespace, db.BlobLogs, db.Rollup, db.State, cfg.Indexer.Name, blobReceiver)
v1.POST("/blob", namespaceHandlers.Blob)

Expand Down Expand Up @@ -530,3 +527,26 @@ func initGasTracker(ctx context.Context, db postgres.Storage) {
}
gasTracker.Start(ctx)
}

func initBlobReceiver(ctx context.Context, cfg Config) (node.DalApi, error) {
switch cfg.ApiConfig.BlobReceiver {
case "r2":
r2 := blob.NewR2(blob.R2Config{
BucketName: os.Getenv("R2_BUCKET"),
AccountId: os.Getenv("R2_ACCOUNT_ID"),
AccessKeyId: os.Getenv("R2_ACCESS_KEY_ID"),
AccessKeySecret: os.Getenv("R2_ACCESS_KEY_SECRET"),
})
err := r2.Init(ctx)
return r2, err
default:
datasource, ok := cfg.DataSources[cfg.ApiConfig.BlobReceiver]
if !ok {
return nil, errors.Errorf("unknown data source pointed in blob_receiver: %s", cfg.ApiConfig.BlobReceiver)
}

return nodeApi.New(datasource.URL).
WithAuthToken(os.Getenv("CELESTIA_NODE_AUTH_TOKEN")).
WithRateLimit(datasource.RequestsPerSecond), nil
}
}
3 changes: 2 additions & 1 deletion configs/dipdup.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ indexer:
threads_count: ${INDEXER_THREADS_COUNT:-1}
block_period: ${INDEXER_BLOCK_PERIOD:-15} # seconds
scripts_dir: ${INDEXER_SCRIPTS_DIR:-./database}
blob_saver: ${INDEXER_BLOB_SAVER}

database:
kind: postgres
Expand Down Expand Up @@ -41,7 +42,7 @@ api:
rate_limit: ${API_RATE_LIMIT:-0}
prometheus: ${API_PROMETHEUS_ENABLED:-true}
request_timeout: ${API_REQUEST_TIMEOUT:-30}
blob_receiver: dal_api
blob_receiver: ${BLOB_RECEIVER:-dal_api}
sentry_dsn: ${SENTRY_DSN}
websocket: ${API_WEBSOCKET_ENABLED:-true}

Expand Down
33 changes: 26 additions & 7 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,22 @@ go 1.22.3
require (
cosmossdk.io/errors v1.0.0
cosmossdk.io/math v1.1.2
github.com/aws/aws-sdk-go-v2 v1.26.1
github.com/aws/aws-sdk-go-v2/config v1.27.11
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
github.com/aws/smithy-go v1.20.2
github.com/celestiaorg/celestia-app v1.10.0
github.com/celestiaorg/go-square v1.0.1
github.com/celestiaorg/go-square/merkle v0.0.0-20240429192549-dea967e1533b
github.com/celestiaorg/rsmt2d v0.11.0
github.com/cosmos/cosmos-sdk v0.46.16
github.com/cosmos/ibc-go/v6 v6.2.2
github.com/dipdup-io/workerpool v0.0.4
github.com/dipdup-net/go-lib v0.4.0
github.com/dipdup-net/indexer-sdk v0.0.5
github.com/fatih/structs v1.1.0
github.com/gabriel-vasile/mimetype v1.4.3
github.com/getsentry/sentry-go v0.27.0
github.com/getsentry/sentry-go/otel v0.27.0
github.com/go-playground/validator/v10 v10.18.0
Expand All @@ -24,7 +31,7 @@ require (
github.com/grafana/pyroscope-go v1.1.1
github.com/json-iterator/go v1.1.12
github.com/labstack/echo-contrib v0.15.0
github.com/labstack/echo/v4 v4.11.3
github.com/labstack/echo/v4 v4.12.0
github.com/lib/pq v1.10.9
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.31.0
Expand Down Expand Up @@ -65,6 +72,19 @@ require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go v1.44.122 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/bgentry/speakeasy v0.1.0 // indirect
Expand Down Expand Up @@ -109,7 +129,6 @@ require (
github.com/ethereum/go-ethereum v1.13.15 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.6.1 // indirect
github.com/go-kit/kit v0.12.0 // indirect
Expand Down Expand Up @@ -164,7 +183,7 @@ require (
github.com/klauspost/compress v1.17.3 // indirect
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
github.com/klauspost/reedsolomon v1.11.8 // indirect
github.com/labstack/gommon v0.4.0 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
Expand Down Expand Up @@ -234,13 +253,13 @@ require (
go.etcd.io/bbolt v1.3.7 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/oauth2 v0.15.0 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.16.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
Expand Down
Loading

0 comments on commit 7f71895

Please sign in to comment.