diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6bd894d6b..85f8a306d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,7 +46,7 @@ jobs: context: . file: build/sandbox/Dockerfile build-args: | - TAG=4.6.0 + TAG=4.6.2 push: true cache-from: type=gha cache-to: type=gha,mode=max diff --git a/Makefile b/Makefile index 72c39ade1..3bea78aa4 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ sandbox-pull: TAG=4.4.0 docker-compose -f docker-compose.flextesa.yml pull flextesa-sandbox: - COMPOSE_PROJECT_NAME=bcdbox TAG=4.4.0 docker-compose -f docker-compose.flextesa.yml up -d + COMPOSE_PROJECT_NAME=bcdbox TAG=4.6.2 docker-compose -f docker-compose.flextesa.yml up -d sandbox-down: COMPOSE_PROJECT_NAME=bcdbox docker-compose -f docker-compose.flextesa.yml down diff --git a/cmd/api/handlers/config.go b/cmd/api/handlers/config.go index 2dc21bdc2..5344001fb 100644 --- a/cmd/api/handlers/config.go +++ b/cmd/api/handlers/config.go @@ -13,16 +13,9 @@ func GetConfig() gin.HandlerFunc { ctxs := c.MustGet("contexts").(config.Contexts) ctx := ctxs.Any() - tzktEndpoints := make(map[string]string) - for network, tzkt := range ctx.Config.TzKT { - tzktEndpoints[network] = tzkt.BaseURI - break - } - cfg := ConfigResponse{ Networks: ctx.Config.API.Networks, RPCEndpoints: ctx.Config.API.Frontend.RPC, - TzKTEndpoints: tzktEndpoints, GaEnabled: ctx.Config.API.Frontend.GaEnabled, MempoolEnabled: ctx.Config.API.Frontend.MempoolEnabled, SandboxMode: ctx.Config.API.Frontend.SandboxMode, diff --git a/cmd/api/handlers/responses.go b/cmd/api/handlers/responses.go index e06b89f94..466e3e8a9 100644 --- a/cmd/api/handlers/responses.go +++ b/cmd/api/handlers/responses.go @@ -483,7 +483,6 @@ type BigMapHistoryItem struct { type ConfigResponse struct { Networks []string `json:"networks"` RPCEndpoints map[string]string `json:"rpc_endpoints"` - TzKTEndpoints map[string]string `json:"tzkt_endpoints"` SentryDSN string `json:"sentry_dsn"` OauthEnabled bool `json:"oauth_enabled"` GaEnabled bool `json:"ga_enabled"` diff --git a/cmd/api/main.go b/cmd/api/main.go index 532d46cc4..05eeb9089 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -15,13 +15,14 @@ import ( "github.com/baking-bad/bcdhub/internal/helpers" "github.com/baking-bad/bcdhub/internal/logger" "github.com/baking-bad/bcdhub/internal/periodic" + "github.com/baking-bad/bcdhub/internal/profiler" "github.com/gin-contrib/cache" "github.com/gin-contrib/cache/persistence" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" - "github.com/pyroscope-io/client/pyroscope" + "github.com/grafana/pyroscope-go" ) type app struct { @@ -52,29 +53,8 @@ func newApp() *app { runtime.SetMutexProfileFraction(5) runtime.SetBlockProfileRate(5) - if cfg.Profiler != nil && cfg.Profiler.Server != "" { - profiler, err := pyroscope.Start(pyroscope.Config{ - ApplicationName: "bcdhub.api", - ServerAddress: cfg.Profiler.Server, - Tags: map[string]string{ - "hostname": os.Getenv("BCDHUB_SERVICE"), - "project": "bcdhub", - "service": "api", - }, - - ProfileTypes: []pyroscope.ProfileType{ - pyroscope.ProfileCPU, - pyroscope.ProfileAllocObjects, - pyroscope.ProfileAllocSpace, - pyroscope.ProfileInuseObjects, - pyroscope.ProfileInuseSpace, - pyroscope.ProfileGoroutines, - pyroscope.ProfileMutexCount, - pyroscope.ProfileMutexDuration, - pyroscope.ProfileBlockCount, - pyroscope.ProfileBlockDuration, - }, - }) + if cfg.Profiler != nil { + profiler, err := profiler.New(cfg.Profiler.Server, "api") if err != nil { panic(err) } diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go index 20c363624..ee042369f 100644 --- a/cmd/indexer/main.go +++ b/cmd/indexer/main.go @@ -10,8 +10,9 @@ import ( "github.com/baking-bad/bcdhub/internal/config" "github.com/baking-bad/bcdhub/internal/helpers" "github.com/baking-bad/bcdhub/internal/logger" + "github.com/baking-bad/bcdhub/internal/profiler" "github.com/dipdup-io/workerpool" - "github.com/pyroscope-io/client/pyroscope" + "github.com/grafana/pyroscope-go" ) func main() { @@ -27,30 +28,9 @@ func main() { defer helpers.CatchPanicSentry() } - var profiler *pyroscope.Profiler - if cfg.Profiler != nil && cfg.Profiler.Server != "" { - profiler, err = pyroscope.Start(pyroscope.Config{ - ApplicationName: "bcdhub.indexer", - ServerAddress: cfg.Profiler.Server, - Tags: map[string]string{ - "hostname": os.Getenv("BCDHUB_SERVICE"), - "project": "bcdhub", - "service": "indexer", - }, - - ProfileTypes: []pyroscope.ProfileType{ - pyroscope.ProfileCPU, - pyroscope.ProfileAllocObjects, - pyroscope.ProfileAllocSpace, - pyroscope.ProfileInuseObjects, - pyroscope.ProfileInuseSpace, - pyroscope.ProfileGoroutines, - pyroscope.ProfileMutexCount, - pyroscope.ProfileMutexDuration, - pyroscope.ProfileBlockCount, - pyroscope.ProfileBlockDuration, - }, - }) + var prof *pyroscope.Profiler + if cfg.Profiler != nil { + prof, err = profiler.New(cfg.Profiler.Server, "indexer") if err != nil { panic(err) } @@ -80,8 +60,8 @@ func main() { } } - if profiler != nil { - if err := profiler.Stop(); err != nil { + if prof != nil { + if err := prof.Stop(); err != nil { panic(err) } } diff --git a/configs/development.yml b/configs/development.yml index 1846e3a4c..2c6d103b6 100644 --- a/configs/development.yml +++ b/configs/development.yml @@ -16,10 +16,6 @@ rpc: timeout: 20 requests_per_second: 20 -tzkt: - mainnet: - uri: https://api.tzkt.io/v1 - services: mainnet: mempool: https://mempool.dipdup.net/v1/graphql @@ -93,11 +89,6 @@ indexer: idle: 5 scripts: - aws: - bucket_name: bcd-elastic-snapshots - region: eu-central-1 - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_SECRET_ACCESS_KEY} networks: - mainnet - ghostnet diff --git a/configs/production.yml b/configs/production.yml index a08013879..62735799d 100644 --- a/configs/production.yml +++ b/configs/production.yml @@ -15,11 +15,6 @@ rpc: uri: https://rpc.tzkt.io/oxfordnet timeout: 20 requests_per_second: 20 - -tzkt: - mainnet: - uri: https://api.tzkt.io/v1 - base_uri: https://api.tzkt.io services: mainnet: diff --git a/configs/testnets.yml b/configs/testnets.yml index c41a37136..d08c34058 100644 --- a/configs/testnets.yml +++ b/configs/testnets.yml @@ -73,11 +73,6 @@ indexer: idle: 5 scripts: - aws: - bucket_name: bcd-elastic-snapshots - region: eu-central-1 - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_SECRET_ACCESS_KEY} networks: - dailynet - mondaynet diff --git a/docs/configuration.md b/docs/configuration.md index b95beaa4d..d895300f4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -15,16 +15,6 @@ rpc: requests_per_second: 10 ``` -#### `tzkt` -TzKT API endpoints (optional) and connection timeouts -```yml -tzkt: - mainnet: - uri: https://api.tzkt.io/v1/ - base_uri: https://api.tzkt.io - timeout: 20 -``` - #### `db` PostgreSQL connection string ```yml @@ -37,6 +27,7 @@ storage: password: ${POSTGRES_PASSWORD} sslmode: disable timeout: 10 + log_queries: ${POSTGRES_LOG_QUERIES:-false} ``` #### `sentry` @@ -105,11 +96,6 @@ indexer: Scripts settings for data migrations and [AWS S3](https://aws.amazon.com/s3/) snapshot registry ```yml scripts: - aws: - bucket_name: bcd-elastic-snapshots - region: eu-central-1 - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_SECRET_ACCESS_KEY} networks: - mainnet connections: diff --git a/go.mod b/go.mod index 3faea472d..a817f0638 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/baking-bad/bcdhub go 1.20 require ( - github.com/aws/aws-sdk-go v1.44.92 github.com/btcsuite/btcutil v1.0.2 github.com/dipdup-io/workerpool v0.0.4 github.com/ebellocchia/go-base58 v0.1.0 @@ -15,6 +14,7 @@ require ( github.com/go-pg/pg/v10 v10.10.6 github.com/go-playground/validator/v10 v10.14.1 github.com/google/uuid v1.3.0 + github.com/grafana/pyroscope-go v1.0.3 github.com/iancoleman/strcase v0.2.0 github.com/ipfs/go-cid v0.3.2 github.com/jessevdk/go-flags v1.5.0 @@ -24,7 +24,6 @@ require ( github.com/machinebox/graphql v0.2.2 github.com/microcosm-cc/bluemonday v1.0.20 github.com/pkg/errors v0.9.1 - github.com/pyroscope-io/client v0.7.2 github.com/robfig/cron/v3 v3.0.1 github.com/rs/zerolog v1.28.0 github.com/sergi/go-diff v1.2.0 @@ -53,8 +52,8 @@ require ( github.com/gomodule/redigo v2.0.0+incompatible // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/gorilla/css v1.0.0 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.4 // indirect github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/karlseguin/expect v1.0.8 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect @@ -74,7 +73,6 @@ require ( github.com/onsi/gomega v1.20.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index 929eef563..f15686997 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/aws/aws-sdk-go v1.44.92 h1:ayc8sQntRMX84Ib9Eqntar7knfNsWHJY7wnZUk5018w= -github.com/aws/aws-sdk-go v1.44.92/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d h1:pVrfxiGfwelyab6n21ZBkbkmbevaf+WvMIiR7sr97hw= @@ -103,6 +101,10 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/grafana/pyroscope-go v1.0.3 h1:8WWmItzLfg4m8G+j//ElSjMeMr88Y6Lvblar6qeTyKk= +github.com/grafana/pyroscope-go v1.0.3/go.mod h1:0d7ftwSMBV/Awm7CCiYmHQEG8Y44Ma3YSjt+nWcWztY= +github.com/grafana/pyroscope-go/godeltaprof v0.1.4 h1:mDsJ3ngul7UfrHibGQpV66PbZ3q1T8glz/tK3bQKKEk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.4/go.mod h1:1HSPtjU8vLG0jE9JrTdzjgFqdJ/VgN7fvxBNq3luJko= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= @@ -113,10 +115,6 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -202,10 +200,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAaj/R8= -github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= -github.com/pyroscope-io/godeltaprof v0.1.2 h1:MdlEmYELd5w+lvIzmZvXGNMVzW2Qc9jDMuJaPOR75g4= -github.com/pyroscope-io/godeltaprof v0.1.2/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62 h1:pyecQtsPmlkCsMkYhT5iZ+sUXuwee+OvfuJjinEA3ko= @@ -290,7 +284,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -316,19 +309,16 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ= @@ -373,9 +363,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/aws/aws.go b/internal/aws/aws.go deleted file mode 100644 index 59155946e..000000000 --- a/internal/aws/aws.go +++ /dev/null @@ -1,63 +0,0 @@ -package aws - -import ( - "bytes" - "io" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/pkg/errors" -) - -// Client - -type Client struct { - Session *session.Session - Bucket string -} - -// New - -func New(id, secret, region, bucket string) (*Client, error) { - sess, err := session.NewSession(&aws.Config{ - Region: aws.String(region), - Credentials: credentials.NewStaticCredentials(id, secret, ""), - }) - if err != nil { - return nil, err - } - - return &Client{ - Session: sess, - Bucket: bucket, - }, nil -} - -// Upload - -func (c *Client) Upload(body io.Reader, filename string) (*s3manager.UploadOutput, error) { - uploader := s3manager.NewUploader(c.Session) - - return uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(c.Bucket), - Key: aws.String(filename), - Body: body, - ContentType: aws.String("application/json"), - }) -} - -// Download - -func (c *Client) Download(filename string) (io.Reader, error) { - downloader := s3manager.NewDownloader(c.Session) - - buf := aws.NewWriteAtBuffer([]byte{}) - - if _, err := downloader.Download(buf, &s3.GetObjectInput{ - Bucket: aws.String(c.Bucket), - Key: aws.String(filename), - }); err != nil { - return nil, errors.Errorf("failed to download file, %v", err) - } - - return bytes.NewReader(buf.Bytes()), nil -} diff --git a/internal/config/config.go b/internal/config/config.go index 785026d81..9ecbfaaf6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,7 +21,6 @@ const ( // Config - type Config struct { RPC map[string]RPCConfig `yaml:"rpc"` - TzKT map[string]TzKTConfig `yaml:"tzkt"` Services map[string]ServiceConfig `yaml:"services"` Storage StorageConfig `yaml:"storage"` Sentry SentryConfig `yaml:"sentry"` @@ -39,7 +38,6 @@ type Config struct { } `yaml:"indexer"` Scripts struct { - AWS AWSConfig `yaml:"aws"` Networks []string `yaml:"networks"` Connections Connections `yaml:"connections"` } `yaml:"scripts"` @@ -109,14 +107,6 @@ type DatabaseConfig struct { Timeout int `yaml:"timeout"` } -// AWSConfig - -type AWSConfig struct { - BucketName string `yaml:"bucket_name"` - Region string `yaml:"region"` - AccessKeyID string `yaml:"access_key_id"` - SecretAccessKey string `yaml:"secret_access_key"` -} - // OAuthConfig - type OAuthConfig struct { State string `yaml:"state"` diff --git a/internal/profiler/profiler.go b/internal/profiler/profiler.go new file mode 100644 index 000000000..bfd9c5fd3 --- /dev/null +++ b/internal/profiler/profiler.go @@ -0,0 +1,36 @@ +package profiler + +import ( + "fmt" + "os" + + "github.com/grafana/pyroscope-go" +) + +func New(server, service string) (*pyroscope.Profiler, error) { + if server == "" { + return nil, nil + } + return pyroscope.Start(pyroscope.Config{ + ApplicationName: fmt.Sprintf("bcdhub.%s", service), + ServerAddress: server, + Tags: map[string]string{ + "hostname": os.Getenv("BCDHUB_SERVICE"), + "project": "bcdhub", + "service": service, + }, + + ProfileTypes: []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + }) +} diff --git a/scripts/migration/main.go b/scripts/migration/main.go index 7fc7a7534..4301bc641 100644 --- a/scripts/migration/main.go +++ b/scripts/migration/main.go @@ -13,12 +13,7 @@ import ( "github.com/pkg/errors" ) -var migrationsList = []migrations.Migration{ - &migrations.DefaultEntrypoint{}, - &migrations.FindLostContracts{}, - &migrations.BigMapStateCount{}, - &migrations.GlobalConstantsRelations{}, -} +var migrationsList = []migrations.Migration{} func main() { migration, err := chooseMigration() diff --git a/scripts/migration/migrations/big_map_state_count.go b/scripts/migration/migrations/big_map_state_count.go deleted file mode 100644 index 55076a55f..000000000 --- a/scripts/migration/migrations/big_map_state_count.go +++ /dev/null @@ -1,58 +0,0 @@ -package migrations - -import ( - "context" - - "github.com/baking-bad/bcdhub/internal/config" - "github.com/baking-bad/bcdhub/internal/models/bigmapdiff" - "github.com/go-pg/pg/v10" -) - -// BigMapStateCount - -type BigMapStateCount struct{} - -// Key - -func (m *BigMapStateCount) Key() string { - return "big_map_state_count" -} - -// Description - -func (m *BigMapStateCount) Description() string { - return "set big map state count" -} - -// Do - migrate function -func (m *BigMapStateCount) Do(ctx *config.Context) error { - var lastID int64 - var end bool - for !end { - if err := ctx.StorageDB.DB.RunInTransaction(context.Background(), func(tx *pg.Tx) error { - var states []bigmapdiff.BigMapState - if err := tx.Model(&bigmapdiff.BigMapState{}).Where("id > ?", lastID).Order("id asc").Limit(10000).Select(&states); err != nil { - return err - } - - for _, state := range states { - count, err := tx.Model(&bigmapdiff.BigMapDiff{}). - Where("ptr = ?", state.Ptr).Where("key_hash = ?", state.KeyHash).Where("contract = ?", state.Contract). - Count() - if err != nil { - return err - } - state.Count = int64(count) - - if _, err := tx.Model(&state).Set("count = ?count").Where("id = ?id").Update(); err != nil { - return err - } - - lastID = state.ID - } - end = len(states)%10000 != 0 - - return nil - }); err != nil { - return err - } - } - return nil -} diff --git a/scripts/migration/migrations/default_entrypoint.go b/scripts/migration/migrations/default_entrypoint.go deleted file mode 100644 index c183f16b2..000000000 --- a/scripts/migration/migrations/default_entrypoint.go +++ /dev/null @@ -1,31 +0,0 @@ -package migrations - -import ( - "context" - - "github.com/baking-bad/bcdhub/internal/bcd/consts" - "github.com/baking-bad/bcdhub/internal/config" - "github.com/baking-bad/bcdhub/internal/models" - "github.com/go-pg/pg/v10" -) - -// DefaultEntrypoint - set entrypoint `default` to contract calls with empty parameters -type DefaultEntrypoint struct{} - -// Key - -func (m *DefaultEntrypoint) Key() string { - return "default_entrypoint" -} - -// Description - -func (m *DefaultEntrypoint) Description() string { - return "set entrypoint `default` to contract calls with empty parameters" -} - -// Do - migrate function -func (m *DefaultEntrypoint) Do(ctx *config.Context) error { - return ctx.StorageDB.DB.RunInTransaction(context.Background(), func(tx *pg.Tx) error { - _, err := tx.Model().Table(models.DocOperations).Where("destination LIKE 'KT1%' AND parameters is null").Update("entrypoint", consts.DefaultEntrypoint) - return err - }) -} diff --git a/scripts/migration/migrations/find_scripts.go b/scripts/migration/migrations/find_scripts.go deleted file mode 100644 index c8614bb7d..000000000 --- a/scripts/migration/migrations/find_scripts.go +++ /dev/null @@ -1,184 +0,0 @@ -package migrations - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - - "github.com/baking-bad/bcdhub/internal/bcd" - astContract "github.com/baking-bad/bcdhub/internal/bcd/contract" - "github.com/baking-bad/bcdhub/internal/config" - "github.com/baking-bad/bcdhub/internal/models/contract" - "github.com/baking-bad/bcdhub/internal/models/types" - "github.com/go-pg/pg/v10" - "github.com/pkg/errors" -) - -// FindLostContracts - -type FindLostContracts struct { - lastID int64 - bulkCount int -} - -// Key - -func (m *FindLostContracts) Key() string { - return "find_lost_script_relations" -} - -// Description - -func (m *FindLostContracts) Description() string { - return "find `scripts` relation" -} - -// Do - migrate function -func (m *FindLostContracts) Do(ctx *config.Context) error { - if m.bulkCount == 0 { - m.bulkCount = 1000 - } - - return ctx.StorageDB.DB.RunInTransaction(context.Background(), func(tx *pg.Tx) error { - var end bool - - for !end { - fmt.Printf("last id = %d\r", m.lastID) - contracts, err := m.getContracts(tx) - if err != nil { - return err - } - - if err := m.findScripts(ctx, tx, contracts); err != nil { - return err - } - - end = len(contracts) != m.bulkCount - - if err := m.saveContracts(tx, contracts); err != nil { - return err - } - } - return nil - }) - -} - -func (m *FindLostContracts) getContracts(db pg.DBI) (resp []contract.Contract, err error) { - query := db.Model((*contract.Contract)(nil)).Order("contract.id asc").Where("alpha_id is NULL and babylon_id is null").Relation("Account.address") - if m.lastID > 0 { - query.Where("contract.id > ?", m.lastID) - } - err = query.Limit(m.bulkCount).Select(&resp) - return -} - -func (m *FindLostContracts) saveContracts(db pg.DBI, contracts []contract.Contract) (err error) { - _, err = db.Model(&contracts).Set("alpha_id = ?alpha_id, babylon_id = ?babylon_id, tags = ?tags").WherePK().Update() - return -} - -func (m *FindLostContracts) findScripts(ctx *config.Context, db pg.DBI, contracts []contract.Contract) error { - for i := range contracts { - scriptBytes, err := ctx.RPC.GetRawScript(context.Background(), contracts[i].Account.Address, 0) - if err != nil { - return err - } - script, err := astContract.NewParser(scriptBytes) - if err != nil { - return errors.Wrap(err, "astContract.NewParser") - } - contractScript, err := ctx.Scripts.ByHash(script.Hash) - if err != nil { - if !ctx.Storage.IsRecordNotFound(err) { - return err - } - var s bcd.RawScript - if err := json.Unmarshal(script.CodeRaw, &s); err != nil { - return err - } - contractScript = contract.Script{ - Hash: script.Hash, - Code: s.Code, - Parameter: s.Parameter, - Storage: s.Storage, - Views: s.Views, - } - - constants, err := script.FindConstants() - if err != nil { - return errors.Wrap(err, "script.FindConstants") - } - - if len(constants) > 0 { - globalConstants, err := ctx.GlobalConstants.All(constants...) - if err != nil { - return err - } - contractScript.Constants = globalConstants - scriptBytes = m.replaceConstants(&contractScript, scriptBytes) - - script, err = astContract.NewParser(scriptBytes) - if err != nil { - return errors.Wrap(err, "astContract.NewParser") - } - - if err := script.Parse(); err != nil { - return err - } - - contractScript.FailStrings = script.FailStrings.Values() - contractScript.Annotations = script.Annotations.Values() - contractScript.Tags = types.NewTags(script.Tags.Values()) - contractScript.Hardcoded = script.HardcodedAddresses.Values() - - params, err := script.Code.Parameter.ToTypedAST() - if err != nil { - return err - } - contractScript.Entrypoints = params.GetEntrypoints() - - if script.IsUpgradable() { - contractScript.Tags.Set(types.UpgradableTag) - } - - if err := contractScript.Save(db); err != nil { - return err - } - - if ctx.Network != types.Mainnet { - contracts[i].BabylonID = contractScript.ID - } else { - if contracts[i].Level <= 655360 { - contracts[i].AlphaID = contractScript.ID - } else { - contracts[i].BabylonID = contractScript.ID - } - } - - } - continue - } - if ctx.Network != types.Mainnet { - contracts[i].BabylonID = contractScript.ID - } else { - if contracts[i].Level <= 655360 { - contracts[i].AlphaID = contractScript.ID - } else { - contracts[i].BabylonID = contractScript.ID - } - } - } - - return nil -} - -func (m *FindLostContracts) replaceConstants(c *contract.Script, script []byte) []byte { - pattern := `{"prim":"constant","args":[{"string":"%s"}]}` - for i := range c.Constants { - script = bytes.ReplaceAll( - script, - []byte(fmt.Sprintf(pattern, c.Constants[i].Address)), - c.Constants[i].Value, - ) - } - return script -} diff --git a/scripts/migration/migrations/global_constants.go b/scripts/migration/migrations/global_constants.go deleted file mode 100644 index fb2feb1fb..000000000 --- a/scripts/migration/migrations/global_constants.go +++ /dev/null @@ -1,116 +0,0 @@ -package migrations - -import ( - "context" - "encoding/json" - "strings" - - "github.com/baking-bad/bcdhub/internal/bcd/ast" - astContract "github.com/baking-bad/bcdhub/internal/bcd/contract" - "github.com/baking-bad/bcdhub/internal/config" - "github.com/baking-bad/bcdhub/internal/logger" - "github.com/baking-bad/bcdhub/internal/models/account" - "github.com/baking-bad/bcdhub/internal/models/contract" -) - -// GlobalConstantsRelations - -type GlobalConstantsRelations struct{} - -// Key - -func (m *GlobalConstantsRelations) Key() string { - return "recover_global_constants" -} - -// Description - -func (m *GlobalConstantsRelations) Description() string { - return "recover `global_constants` relation" -} - -// Do - migrate function -func (m *GlobalConstantsRelations) Do(ctx *config.Context) error { - var offset int - var end bool - for !end { - scripts, err := ctx.Scripts.GetScripts(10, offset) - if err != nil { - if strings.Contains(err.Error(), "no rows in result set") { - end = true - continue - } - return err - } - - for i := range scripts { - var accountID int64 - if err := ctx.StorageDB.DB.Model(&contract.Contract{}). - Column("account_id"). - Where("jakarta_id = ?", scripts[i].ID). - WhereOr("babylon_id = ?", scripts[i].ID). - OrderExpr("id ASC").Limit(1). - Select(&accountID); err != nil { - if strings.Contains(err.Error(), "no rows in result set") { - continue - } - return err - } - - var address string - if err := ctx.StorageDB.DB.Model(&account.Account{}). - Column("address"). - Where("id = ?", accountID). - OrderExpr("id ASC").Limit(1). - Select(&address); err != nil { - if strings.Contains(err.Error(), "no rows in result set") { - continue - } - return err - } - - logger.Info().Str("address", address).Msg("finding constants...") - - data, err := ctx.RPC.GetRawScript(context.Background(), address, 0) - if err != nil { - return err - } - - var cd astContract.ContractData - if err := json.Unmarshal(data, &cd); err != nil { - return err - } - - var tree ast.UntypedAST - if err := json.Unmarshal(cd.Code, &tree); err != nil { - return err - } - - constants, err := astContract.FindConstants(tree) - if err != nil { - return err - } - - if len(constants) == 0 { - continue - } - logger.Info().Str("address", address).Int("constants_count", len(constants)).Msg("found constants") - - for key := range constants { - gc, err := ctx.GlobalConstants.Get(key) - if err != nil { - return err - } - relation := contract.ScriptConstants{ - GlobalConstantId: gc.ID, - ScriptId: scripts[i].ID, - } - - if _, err := ctx.StorageDB.DB.Model(&relation).Insert(); err != nil { - return err - } - } - } - - offset += len(scripts) - end = len(scripts) == 0 - } - return nil -} diff --git a/version.json b/version.json index c18f4bbdd..c4dcb6947 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { "name": "bcdhub", - "version": "4.6.0" + "version": "4.7.2" }