Skip to content

Commit

Permalink
Merge branch 'stage' into alan/fork-cleanup
Browse files Browse the repository at this point in the history
# Conflicts:
#	network/topics/pubsub.go
#	operator/validator/controller.go
  • Loading branch information
nkryuchkov committed Oct 31, 2024
2 parents c182dc8 + 8a5bcfc commit 295f728
Show file tree
Hide file tree
Showing 36 changed files with 663 additions and 208 deletions.
86 changes: 0 additions & 86 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,6 @@ variables:
IMAGE_NAME: ssv-node
DOCKER_BUILDKIT: 1

#STAGE
ACCOUNT_ID_INFRA_STAGE: 121827225315
AWS_REGION_INFRA_STAGE: "us-west-2"
DOCKER_REPO_INFRA_STAGE: $ACCOUNT_ID_INFRA_STAGE.dkr.ecr.$AWS_REGION_INFRA_STAGE.amazonaws.com/$IMAGE_NAME
APP_REPLICAS_INFRA_STAGE: "1"
ECRLOGIN_INFRA_STAGE: "aws ecr get-login --registry-ids $ACCOUNT_ID_INFRA_STAGE --region $AWS_REGION_INFRA_STAGE --no-include-email"
STAGE_HEALTH_CHECK_IMAGE: 121827225315.dkr.ecr.us-west-2.amazonaws.com/infra-stage-repo:ubuntu20

#PRODUCTION
ACCOUNT_ID_INFRA_PROD: 764289642555
AWS_REGION_INFRA_PROD: "us-west-2"
Expand All @@ -23,84 +15,6 @@ variables:
ECRLOGIN_INFRA_PROD: "aws ecr get-login --registry-ids $ACCOUNT_ID_INFRA_PROD --region $AWS_REGION_INFRA_PROD --no-include-email"
PROD_HEALTH_CHECK_IMAGE: 764289642555.dkr.ecr.us-west-2.amazonaws.com/infra-prod-repo:ubuntu20

# +-------+
# | STAGE |
# +-------+
Build stage Docker image:
image: docker:20.10.23
stage: build
tags:
- blox-infra-stage
script:
- apk add --no-cache py-pip
- pip install pyyaml==5.3.1
- pip install awscli
- docker build -t $IMAGE_NAME:$CI_COMMIT_SHA -f Dockerfile .
- DOCKER_LOGIN_TO_INFRA_STAGE_REPO=`$ECRLOGIN_INFRA_STAGE`
- docker tag $IMAGE_NAME:$CI_COMMIT_SHA $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
- $DOCKER_LOGIN_TO_INFRA_STAGE_REPO && docker push $DOCKER_REPO_INFRA_STAGE:$CI_COMMIT_SHA
only:
- stage

# +---------------------+
# | STAGE HETZNER NODES |
# +---------------------+


Deploy nodes to hetzner stage:
stage: deploy
tags:
- hetzner-k8s-stage
image: bitnami/kubectl:1.27.5
script:
- export K8S_API_VERSION=$INFRA_STAGE_K8_API_VERSION
- export SSV_NODES_CPU_LIMIT=$HETZNER_STAGE_SSV_NODES_CPU_LIMIT
- export SSV_NODES_MEM_LIMIT=$HETZNER_STAGE_SSV_NODES_MEM_LIMIT
- echo $HETZNER_KUBECONFIG | base64 -d > kubeconfig
- mv kubeconfig ~/.kube/
- export KUBECONFIG=~/.kube/kubeconfig
- kubectl config get-contexts
#
# +--------------------+
# | Deploy SSV nodes |
# +--------------------+
- .k8/hetzner-stage/scripts/deploy-cluster-1--4.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-5--8.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-9--12.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-13--16.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-17--20.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-21--24.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-25--28.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-29--32.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-33--36.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-37--40.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-41--44.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-45--48.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-49--52.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-53--56.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-57--60.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-61--64.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-65--68.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
- .k8/hetzner-stage/scripts/deploy-cluster-69--72.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $STAGE_HEALTH_CHECK_IMAGE $SSV_NODES_CPU_LIMIT $SSV_NODES_MEM_LIMIT
only:
- stage

Deploy exporter to hetzner stage:
stage: deploy
tags:
- hetzner-k8s-stage
image: bitnami/kubectl:1.27.5
script:
- export K8S_API_VERSION=$INFRA_STAGE_K8_API_VERSION
- export SSV_EXPORTER_CPU_LIMIT=$STAGE_SSV_EXPORTER_CPU_LIMIT
- export SSV_EXPORTER_MEM_LIMIT=$STAGE_SSV_EXPORTER_MEM_LIMIT
- echo $HETZNER_KUBECONFIG | base64 -d > kubeconfig
- mv kubeconfig ~/.kube/
- export KUBECONFIG=~/.kube/kubeconfig
- kubectl config get-contexts
- .k8/hetzner-stage/scripts/deploy-holesky-exporters.sh $DOCKER_REPO_INFRA_STAGE $CI_COMMIT_SHA ssv $APP_REPLICAS_INFRA_STAGE hetzner.stage.k8s.local hetzner.stage.k8s.local stage.ssv.network $K8S_API_VERSION $SSV_EXPORTER_CPU_LIMIT $SSV_EXPORTER_MEM_LIMIT
only:
- stage

# +---------------+
# | Prod |
Expand Down
6 changes: 6 additions & 0 deletions api/bind.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@ func Bind(r *http.Request, dest interface{}) error {
return err
}
fieldValue.SetInt(v)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
v, err := strconv.ParseUint(formValue, 10, 64)
if err != nil {
return err
}
fieldValue.SetUint(v)
case reflect.Float32, reflect.Float64:
v, err := strconv.ParseFloat(formValue, 64)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion api/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func (e *ErrorResponse) Error() string {
return e.Err.Error()
}

func InvalidRequestError(err error) *ErrorResponse {
func BadRequestError(err error) *ErrorResponse {
return &ErrorResponse{
Err: err,
Code: 400,
Expand Down
119 changes: 119 additions & 0 deletions api/handlers/exporter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
package handlers

import (
"fmt"
"net/http"

"github.com/attestantio/go-eth2-client/spec/phase0"
spectypes "github.com/ssvlabs/ssv-spec/types"

"github.com/ssvlabs/ssv/api"
exporterapi "github.com/ssvlabs/ssv/exporter/api"
"github.com/ssvlabs/ssv/exporter/convert"
ibftstorage "github.com/ssvlabs/ssv/ibft/storage"
qbftstorage "github.com/ssvlabs/ssv/protocol/v2/qbft/storage"
"github.com/ssvlabs/ssv/utils/casts"
)

type Exporter struct {
DomainType spectypes.DomainType
QBFTStores *ibftstorage.QBFTStores
}

type ParticipantResponse struct {
Role string `json:"role"`
Slot uint64 `json:"slot"`
PublicKey string `json:"public_key"`
Message struct {
// We're keeping "Signers" capitalized to avoid breaking existing clients that rely on the current structure
Signers []uint64 `json:"Signers"`
} `json:"message"`
}

func (e *Exporter) Decideds(w http.ResponseWriter, r *http.Request) error {
var request struct {
From uint64 `json:"from"`
To uint64 `json:"to"`
Roles api.RoleSlice `json:"roles"`
PubKeys api.HexSlice `json:"pubkeys"`
}
var response struct {
Data []*ParticipantResponse `json:"data"`
}

if err := api.Bind(r, &request); err != nil {
return api.BadRequestError(err)
}

if request.From > request.To {
return api.BadRequestError(fmt.Errorf("'from' must be less than or equal to 'to'"))
}

if len(request.PubKeys) == 0 {
return api.BadRequestError(fmt.Errorf("at least one public key is required"))
}

if len(request.Roles) == 0 {
return api.BadRequestError(fmt.Errorf("at least one role is required"))
}

response.Data = []*ParticipantResponse{}

qbftStores := make(map[convert.RunnerRole]qbftstorage.QBFTStore, len(request.Roles))
for _, role := range request.Roles {
runnerRole := casts.BeaconRoleToConvertRole(spectypes.BeaconRole(role))
storage := e.QBFTStores.Get(runnerRole)
if storage == nil {
return api.Error(fmt.Errorf("role storage doesn't exist: %v", role))
}

qbftStores[runnerRole] = storage
}

for _, role := range request.Roles {
runnerRole := casts.BeaconRoleToConvertRole(spectypes.BeaconRole(role))
qbftStore := qbftStores[runnerRole]

for _, pubKey := range request.PubKeys {
msgID := convert.NewMsgID(e.DomainType, pubKey, runnerRole)
from := phase0.Slot(request.From)
to := phase0.Slot(request.To)

participantsList, err := qbftStore.GetParticipantsInRange(msgID, from, to)
if err != nil {
return api.Error(fmt.Errorf("error getting participants: %w", err))
}

if len(participantsList) == 0 {
continue
}

data, err := exporterapi.ParticipantsAPIData(participantsList...)
if err != nil {
return api.Error(fmt.Errorf("error getting participants API data: %w", err))
}

apiData, ok := data.([]*exporterapi.ParticipantsAPI)
if !ok {
return api.Error(fmt.Errorf("invalid type for participants API data"))
}

for _, apiMsg := range apiData {
response.Data = append(response.Data, transformToParticipantResponse(apiMsg))
}
}
}

return api.Render(w, r, response)
}

func transformToParticipantResponse(apiMsg *exporterapi.ParticipantsAPI) *ParticipantResponse {
response := &ParticipantResponse{
Role: apiMsg.Role,
Slot: uint64(apiMsg.Slot),
PublicKey: apiMsg.ValidatorPK,
}
response.Message.Signers = apiMsg.Signers

return response
}
24 changes: 20 additions & 4 deletions api/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/ssvlabs/ssv/api"
"github.com/ssvlabs/ssv/api/handlers"
"github.com/ssvlabs/ssv/utils/commons"
)

type Server struct {
Expand All @@ -19,19 +20,22 @@ type Server struct {

node *handlers.Node
validators *handlers.Validators
exporter *handlers.Exporter
}

func New(
logger *zap.Logger,
addr string,
node *handlers.Node,
validators *handlers.Validators,
exporter *handlers.Exporter,
) *Server {
return &Server{
logger: logger,
addr: addr,
node: node,
validators: validators,
exporter: exporter,
}
}

Expand All @@ -41,20 +45,25 @@ func (s *Server) Run() error {
router.Use(middleware.Throttle(runtime.NumCPU() * 4))
router.Use(middleware.Compress(5, "application/json"))
router.Use(middlewareLogger(s.logger))
router.Use(middlewareNodeVersion)

router.Get("/v1/node/identity", api.Handler(s.node.Identity))
router.Get("/v1/node/peers", api.Handler(s.node.Peers))
router.Get("/v1/node/topics", api.Handler(s.node.Topics))
router.Get("/v1/node/health", api.Handler(s.node.Health))
router.Get("/v1/validators", api.Handler(s.validators.List))
// We kept both GET and POST methods to ensure compatibility and avoid breaking changes for clients that may rely on either method
router.Get("/v1/exporter/decideds", api.Handler(s.exporter.Decideds))
router.Post("/v1/exporter/decideds", api.Handler(s.exporter.Decideds))

s.logger.Info("Serving SSV API", zap.String("addr", s.addr))

server := &http.Server{
Addr: s.addr,
Handler: router,
ReadTimeout: 12 * time.Second,
WriteTimeout: 12 * time.Second,
Addr: s.addr,
Handler: router,
ReadHeaderTimeout: 10 * time.Second,
ReadTimeout: 12 * time.Second,
WriteTimeout: 12 * time.Second,
}
return server.ListenAndServe()
}
Expand All @@ -80,3 +89,10 @@ func middlewareLogger(logger *zap.Logger) func(next http.Handler) http.Handler {
return http.HandlerFunc(fn)
}
}

func middlewareNodeVersion(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-SSV-Node-Version", commons.GetNodeVersion())
next.ServeHTTP(w, r)
})
}
55 changes: 49 additions & 6 deletions api/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,14 @@ package api

import (
"encoding/hex"
"encoding/json"
"errors"
"strconv"
"strings"

spectypes "github.com/ssvlabs/ssv-spec/types"

"github.com/ssvlabs/ssv/protocol/v2/message"
)

type Hex []byte
Expand All @@ -17,18 +22,15 @@ func (h *Hex) UnmarshalJSON(data []byte) error {
if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' {
return errors.New("invalid hex string")
}
b, err := hex.DecodeString(string(data[1 : len(data)-1]))
if err != nil {
return err
}
*h = b
return nil
str := string(data[1 : len(data)-1])
return h.Bind(str)
}

func (h *Hex) Bind(value string) error {
if value == "" {
return nil
}
value = strings.TrimPrefix(value, "0x")
b, err := hex.DecodeString(value)
if err != nil {
return err
Expand Down Expand Up @@ -69,3 +71,44 @@ func (us *Uint64Slice) Bind(value string) error {
}
return nil
}

type Role spectypes.BeaconRole

func (r *Role) Bind(value string) error {
role, err := message.BeaconRoleFromString(value)
if err != nil {
return err
}
*r = Role(role)
return nil
}

func (r Role) MarshalJSON() ([]byte, error) {
return []byte(`"` + spectypes.BeaconRole(r).String() + `"`), nil
}

func (r *Role) UnmarshalJSON(data []byte) error {
var role string
err := json.Unmarshal(data, &role)
if err != nil {
return err
}
return r.Bind(role)
}

type RoleSlice []Role

func (rs *RoleSlice) Bind(value string) error {
if value == "" {
return nil
}
for _, s := range strings.Split(value, ",") {
var r Role
err := r.Bind(s)
if err != nil {
return err
}
*rs = append(*rs, r)
}
return nil
}
Loading

0 comments on commit 295f728

Please sign in to comment.