diff --git a/config/sequencer/presets/config-batcher.json b/config/sequencer/presets/config-batcher.json index 43508b8211..e8d0fa511c 100644 --- a/config/sequencer/presets/config-batcher.json +++ b/config/sequencer/presets/config-batcher.json @@ -4,4 +4,5 @@ "strk_fee_token_address": "0x7", "batcher_config.storage.db_config.path_prefix": "/data", "batcher_config.storage.db_config.enforce_file_exists": false, + "validator_id" : "0x1" } diff --git a/deployments/images/base/Dockerfile b/deployments/images/base/Dockerfile index 931544cd0c..81da837337 100644 --- a/deployments/images/base/Dockerfile +++ b/deployments/images/base/Dockerfile @@ -7,7 +7,6 @@ # More info on Cargo Chef: https://github.com/LukeMathWalker/cargo-chef FROM ubuntu:22.04 AS base -# WORKDIR /app COPY scripts/install_build_tools.sh . COPY scripts/dependencies.sh . diff --git a/deployments/images/sequencer/Dockerfile b/deployments/images/sequencer/Dockerfile index d62dc01880..73722ad0c3 100644 --- a/deployments/images/sequencer/Dockerfile +++ b/deployments/images/sequencer/Dockerfile @@ -1,21 +1,22 @@ -# syntax = devthefuture/dockerfile-x +#syntax = devthefuture/dockerfile-x INCLUDE deployments/images/base/Dockerfile # Compile the sequencer_node crate in release mode, ensuring dependencies are locked. FROM base AS builder +WORKDIR /app COPY . . -RUN cargo build --release --package starknet_sequencer_node +RUN cargo build --bin starknet_sequencer_node FROM base AS sequencer ENV ID=1000 WORKDIR /app -COPY --from=builder /target/release/starknet_sequencer_node /app/target/release/starknet_sequencer_node +COPY --from=builder /app/target/debug/starknet_sequencer_node ./target/debug/starknet_sequencer_node # Copy sequencer config -COPY config/sequencer config/sequencer +COPY config/sequencer/default_config.json /config/sequencer/ # Create a new user "sequencer". RUN set -ex; \ @@ -30,4 +31,4 @@ EXPOSE 8080 8081 8082 USER ${ID} # Set the entrypoint to use tini to manage the process. -ENTRYPOINT ["tini", "--", "/app/target/release/starknet_sequencer_node"] +ENTRYPOINT ["tini", "--", "/app/target/debug/starknet_sequencer_node"] diff --git a/deployments/sequencer/.gitignore b/deployments/sequencer/.gitignore new file mode 100644 index 0000000000..74a29c3151 --- /dev/null +++ b/deployments/sequencer/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +.idea/ +services/test.py diff --git a/deployments/sequencer/app/service.py b/deployments/sequencer/app/service.py index 5cd5c95fa1..51fc9db775 100644 --- a/deployments/sequencer/app/service.py +++ b/deployments/sequencer/app/service.py @@ -1,12 +1,12 @@ import json +import typing -from itertools import chain from constructs import Construct -from cdk8s import Names, ApiObjectMetadata +from cdk8s import Names from imports import k8s -from imports.com.google import cloud as google +from imports.k8s import IngressTls -from services import topology +from services import topology, const class ServiceApp(Construct): @@ -16,252 +16,207 @@ def __init__( id: str, *, namespace: str, - topology: topology.ServiceTopology, + service_topology: topology.ServiceTopology, ): super().__init__(scope, id) self.namespace = namespace self.label = {"app": Names.to_label_value(self, include_hash=False)} - self.topology = topology - self.node_config = topology.config.get_config() + self.host = f"{self.node.id}.{self.namespace}.sw-dev.io" + self.service_topology = service_topology + self.node_config = service_topology.config.get_merged_config() - self.set_k8s_namespace() + k8s.KubeNamespace(self, "namespace", metadata=k8s.ObjectMeta(name=self.namespace)) - if topology.service is not None: - self.set_k8s_service() - - if topology.config is not None: - self.set_k8s_configmap() - - if topology.deployment is not None: - self.set_k8s_deployment() - - if topology.ingress is not None: - self.set_k8s_ingress() - - if topology.pvc is not None: - self.set_k8s_pvc() - - def set_k8s_namespace(self): - return k8s.KubeNamespace(self, "namespace", metadata=k8s.ObjectMeta(name=self.namespace)) - - def set_k8s_configmap(self): - return k8s.KubeConfigMap( + k8s.KubeConfigMap( self, "configmap", metadata=k8s.ObjectMeta(name=f"{self.node.id}-config"), - data=dict(config=json.dumps(self.topology.config.get_config())), + data=dict(config=json.dumps(self.service_topology.config.get_config())), ) - def set_k8s_service(self): - return k8s.KubeService( + k8s.KubeService( self, "service", spec=k8s.ServiceSpec( - type=self.topology.service.type.value, - ports=[ - k8s.ServicePort( - name=port.name, - port=port.port, - target_port=k8s.IntOrString.from_number(port.container_port), - ) - for port in self.topology.service.ports - ], + type=const.ServiceType.CLUSTER_IP, + ports=self._get_service_ports(), selector=self.label, ), ) - def _get_container_ports(self): - ports = [] - for c in ["http_server_config.port", "monitoring_endpoint_config.port"]: - ports.append(self.node_config[c].get("value")) - return ports - - def set_k8s_deployment(self): - node_http_port = self.node_config["http_server_config.port"].get("value") - return k8s.KubeDeployment( + k8s.KubeDeployment( self, "deployment", spec=k8s.DeploymentSpec( - replicas=self.topology.deployment.replicas, + replicas=self.service_topology.deployment.replicas, selector=k8s.LabelSelector(match_labels=self.label), template=k8s.PodTemplateSpec( metadata=k8s.ObjectMeta(labels=self.label), spec=k8s.PodSpec( security_context=k8s.PodSecurityContext(fs_group=1000), + volumes=self._get_volumes(), containers=[ k8s.Container( - name=f"{self.node.id}-{container.name}", - image=container.image, + name=self.node.id, + image=self.service_topology.images.get("sequencer"), + image_pull_policy="IfNotPresent", # command=["sleep", "infinity"], - args=container.args, - ports=[ - k8s.ContainerPort( - container_port=port, - ) - for port in self._get_container_ports() - ], - startup_probe=( - k8s.Probe( - http_get=k8s.HttpGetAction( - path=container.startup_probe.path, - port=k8s.IntOrString.from_number(node_http_port), - ), - period_seconds=container.startup_probe.period_seconds, - failure_threshold=container.startup_probe.failure_threshold, - timeout_seconds=container.startup_probe.timeout_seconds, - ) - if container.startup_probe is not None - else None - ), - readiness_probe=( - k8s.Probe( - http_get=k8s.HttpGetAction( - path=container.readiness_probe.path, - port=k8s.IntOrString.from_number(node_http_port), - ), - period_seconds=container.readiness_probe.period_seconds, - failure_threshold=container.readiness_probe.failure_threshold, - timeout_seconds=container.readiness_probe.timeout_seconds, - ) - if container.readiness_probe is not None - else None - ), - liveness_probe=( - k8s.Probe( - http_get=k8s.HttpGetAction( - path=container.liveness_probe.path, - port=k8s.IntOrString.from_number(node_http_port), - ), - period_seconds=container.liveness_probe.period_seconds, - failure_threshold=container.liveness_probe.failure_threshold, - timeout_seconds=container.liveness_probe.timeout_seconds, - ) - if container.liveness_probe is not None - else None - ), - volume_mounts=[ - k8s.VolumeMount( - name=mount.name, - mount_path=mount.mount_path, - read_only=mount.read_only, - ) - for mount in container.volume_mounts - ], + args=const.CONTAINER_ARGS, + ports=self._get_container_ports(), + startup_probe=self._get_http_probe(), + readiness_probe=self._get_http_probe(), + liveness_probe=self._get_http_probe(), + volume_mounts=self._get_volume_mounts(), ) - for container in self.topology.deployment.containers ], - volumes=list( - chain( - ( - ( - k8s.Volume( - name=f"{self.node.id}-{volume.name}", - config_map=k8s.ConfigMapVolumeSource( - name=f"{self.node.id}-{volume.name}" - ), - ) - for volume in self.topology.deployment.configmap_volumes - ) - if self.topology.deployment.configmap_volumes is not None - else None - ), - ( - ( - k8s.Volume( - name=f"{self.node.id}-{volume.name}", - persistent_volume_claim=k8s.PersistentVolumeClaimVolumeSource( - claim_name=f"{self.node.id}-{volume.name}", - read_only=volume.read_only, - ), - ) - for volume in self.topology.deployment.pvc_volumes - ) - if self.topology.deployment is not None - else None - ), - ) - ), ), ), ), ) - def set_k8s_ingress(self): - return k8s.KubeIngress( + k8s.KubeIngress( self, "ingress", metadata=k8s.ObjectMeta( name=f"{self.node.id}-ingress", labels=self.label, - annotations=self.topology.ingress.annotations, + annotations={ + "kubernetes.io/tls-acme": "true", + "cert-manager.io/common-name": self.host, + "cert-manager.io/issue-temporary-certificate": "true", + "cert-manager.io/issuer": "letsencrypt-prod", + "acme.cert-manager.io/http01-edit-in-place": "true", + }, ), spec=k8s.IngressSpec( - ingress_class_name=self.topology.ingress.class_name, - tls=[ - k8s.IngressTls(hosts=tls.hosts, secret_name=tls.secret_name) - for tls in self.topology.ingress.tls or [] - ], - rules=[ - k8s.IngressRule( - host=rule.host, - http=k8s.HttpIngressRuleValue( - paths=[ - k8s.HttpIngressPath( - path=path.path, - path_type=path.path_type, - backend=k8s.IngressBackend( - service=k8s.IngressServiceBackend( - name=path.backend_service_name, - port=k8s.ServiceBackendPort( - number=path.backend_service_port_number - ), - ) - ), - ) - for path in rule.paths or [] - ] - ), - ) - for rule in self.topology.ingress.rules or [] - ], + ingress_class_name="premium-rwo", + tls=self._get_ingress_tls(), + rules=self._get_ingress_rules() ), ) - def set_k8s_pvc(self): k8s.KubePersistentVolumeClaim( self, "pvc", metadata=k8s.ObjectMeta(name=f"{self.node.id}-data", labels=self.label), spec=k8s.PersistentVolumeClaimSpec( - storage_class_name=self.topology.pvc.storage_class_name, - access_modes=self.topology.pvc.access_modes, - volume_mode=self.topology.pvc.volume_mode, + storage_class_name=self.service_topology.pvc.storage_class_name, + access_modes=self.service_topology.pvc.access_modes, + volume_mode=self.service_topology.pvc.volume_mode, resources=k8s.ResourceRequirements( - requests={"storage": k8s.Quantity.from_string(self.topology.pvc.storage)} + requests={"storage": k8s.Quantity.from_string(self.service_topology.pvc.storage)} ), ), ) - def set_k8s_backend_config(self): - return google.BackendConfig( + + def _get_config_attr(self, attribute): + config_attr = self.node_config.get(attribute).get('value') + if config_attr is None: + assert f'Config attribute "{attribute}" is missing.' + else: + return config_attr + + def _get_container_ports(self): + return [ + k8s.ContainerPort( + container_port=self._get_config_attr(port) + ) for port in ["http_server_config.port", "monitoring_endpoint_config.port"] + ] + + def _get_container_resources(self): # TODO: implement method to calc resources based on config + pass + + def _get_service_ports(self): + return [ + k8s.ServicePort( + name=attr.split("_")[0], + port=self._get_config_attr(attr), + target_port=k8s.IntOrString.from_number(self._get_config_attr(attr)) + ) for attr in ["http_server_config.port", "monitoring_endpoint_config.port"] + ] + + def _get_http_probe( self, - "backendconfig", - metadata=ApiObjectMetadata(name=f"{self.node.id}-backendconfig", labels=self.label), - spec=google.BackendConfigSpec( - health_check=google.BackendConfigSpecHealthCheck( - check_interval_sec=5, - healthy_threshold=10, - unhealthy_threshold=5, - timeout_sec=5, - request_path="/", - type="http", - ), - iap=google.BackendConfigSpecIap( - enabled=True, - oauthclient_credentials=google.BackendConfigSpecIapOauthclientCredentials( - secret_name="" - ), - ), + period_seconds: int = const.PROBE_PERIOD_SECONDS, + failure_threshold: int = const.PROBE_FAILURE_THRESHOLD, + timeout_seconds: int = const.PROBE_TIMEOUT_SECONDS + ): + path = "/monitoring/alive" + # path = self.node_config['monitoring_path'].get("value") # TODO add monitoring path in node_config + port = self.node_config.get('monitoring_endpoint_config.port').get("value") + + return k8s.Probe( + http_get=k8s.HttpGetAction( + path=path, + port=k8s.IntOrString.from_number(port), ), + period_seconds=period_seconds, + failure_threshold=failure_threshold, + timeout_seconds=timeout_seconds, ) + + def _get_volume_mounts(self) -> typing.List[k8s.VolumeMount]: + return [ + k8s.VolumeMount( + name=f"{self.node.id}-config", + mount_path="/config/sequencer/presets/", + read_only=True + ), + k8s.VolumeMount( + name=f"{self.node.id}-data", + mount_path="/data", + read_only=False + ) + ] + + def _get_volumes(self): + return [ + k8s.Volume( + name=f"{self.node.id}-config", + config_map=k8s.ConfigMapVolumeSource( + name=f"{self.node.id}-config" + ) + ), + k8s.Volume( + name=f"{self.node.id}-data", + persistent_volume_claim=k8s.PersistentVolumeClaimVolumeSource( + claim_name=f"{self.node.id}-data", + read_only=False + ) + ) + ] + + def _get_ingress_rules(self) -> typing.List[k8s.IngressRule]: + return [ + k8s.IngressRule( + host=self.host, + http=k8s.HttpIngressRuleValue( + paths=[ + k8s.HttpIngressPath( + path="/monitoring", + path_type="Prefix", + backend=k8s.IngressBackend( + service=k8s.IngressServiceBackend( + name=f"{self.node.id}-service", + port=k8s.ServiceBackendPort( + number=self._get_config_attr("monitoring_endpoint_config.port") + ), + ) + ), + ) + ] + ), + ) + ] + + def _get_ingress_tls(self) -> typing.List[IngressTls]: + return [ + k8s.IngressTls( + hosts=[self.host], + secret_name=f"{self.node.id}-tls" + ) + ] + + diff --git a/deployments/sequencer/config/sequencer.py b/deployments/sequencer/config/sequencer.py index 2afb9286a1..f6c619e509 100644 --- a/deployments/sequencer/config/sequencer.py +++ b/deployments/sequencer/config/sequencer.py @@ -10,7 +10,7 @@ class SequencerDevConfig(Config): - def __init__(self, mount_path: str, config_file_path: str = ""): + def __init__(self, config_file_path: str = ""): super().__init__( global_config=json.loads( open(os.path.join(CONFIG_DIR, "default_config.json"), "r").read() @@ -19,8 +19,7 @@ def __init__(self, mount_path: str, config_file_path: str = ""): json.loads(open(os.path.join(CONFIG_DIR, "presets", "config.json"), "r").read()) if not config_file_path else json.loads(open(os.path.abspath(config_file_path)).read()) - ), - mount_path=mount_path, + ) ) def validate(self): diff --git a/deployments/sequencer/imports/com/google/cloud/_jsii/comgooglecloud@0.0.0.jsii.tgz b/deployments/sequencer/imports/com/google/cloud/_jsii/comgooglecloud@0.0.0.jsii.tgz index 01017631ca..549ce98511 100644 Binary files a/deployments/sequencer/imports/com/google/cloud/_jsii/comgooglecloud@0.0.0.jsii.tgz and b/deployments/sequencer/imports/com/google/cloud/_jsii/comgooglecloud@0.0.0.jsii.tgz differ diff --git a/deployments/sequencer/imports/k8s/_jsii/k8s@0.0.0.jsii.tgz b/deployments/sequencer/imports/k8s/_jsii/k8s@0.0.0.jsii.tgz index 176fdafb31..6382fb2a82 100644 Binary files a/deployments/sequencer/imports/k8s/_jsii/k8s@0.0.0.jsii.tgz and b/deployments/sequencer/imports/k8s/_jsii/k8s@0.0.0.jsii.tgz differ diff --git a/deployments/sequencer/main.py b/deployments/sequencer/main.py index 932a6bbd36..9b71507c4c 100644 --- a/deployments/sequencer/main.py +++ b/deployments/sequencer/main.py @@ -25,10 +25,10 @@ def __post_init__(self): class SequencerNode(Chart): def __init__( - self, scope: Construct, name: str, namespace: str, topology: topology.ServiceTopology + self, scope: Construct, name: str, namespace: str, service_topology: topology.ServiceTopology ): super().__init__(scope, name, disable_resource_name_hashes=True, namespace=namespace) - self.service = ServiceApp(self, name, namespace=namespace, topology=topology) + self.service = ServiceApp(self, name, namespace=namespace, service_topology=service_topology) def main(): @@ -45,7 +45,7 @@ def main(): scope=app, name="sequencer-node", namespace=args.namespace, - topology=system_preset, + service_topology=system_preset, ) app.synth() diff --git a/deployments/sequencer/services/const.py b/deployments/sequencer/services/const.py index 6b58abbdd8..a6a956b7b8 100644 --- a/deployments/sequencer/services/const.py +++ b/deployments/sequencer/services/const.py @@ -1,7 +1,7 @@ from enum import Enum # k8s service types -class ServiceType(Enum): +class ServiceType(str, Enum): CLUSTER_IP = "ClusterIP" LOAD_BALANCER = "LoadBalancer" NODE_PORT = "NodePort" @@ -14,5 +14,11 @@ class ServiceType(Enum): # k8s service ports HTTP_SERVICE_PORT = 80 -RPC_SERVICE_PORT = 8081 +GRPC_SERVICE_PORT = 8081 MONITORING_SERVICE_PORT = 8082 + +PROBE_FAILURE_THRESHOLD = 5 +PROBE_PERIOD_SECONDS = 10 +PROBE_TIMEOUT_SECONDS = 5 + +CONTAINER_ARGS = ["--config_file", "/config/sequencer/presets/config"] diff --git a/deployments/sequencer/services/objects.py b/deployments/sequencer/services/objects.py index ece7c20253..d7435e4880 100644 --- a/deployments/sequencer/services/objects.py +++ b/deployments/sequencer/services/objects.py @@ -43,16 +43,18 @@ class PersistentVolumeClaim: class Config: global_config: Dict[Any, Any] config: Dict[Any, Any] - mount_path: str def _merged_config(self) -> Dict[Any, Any]: _config = self.global_config.copy() _config.update(self.config) return _config - def get_config(self): + def get_merged_config(self): return self._merged_config() + def get_config(self): + return self.config + def validate(self): pass diff --git a/deployments/sequencer/services/topology.py b/deployments/sequencer/services/topology.py index 68c3056193..32088d68b3 100644 --- a/deployments/sequencer/services/topology.py +++ b/deployments/sequencer/services/topology.py @@ -7,6 +7,9 @@ @dataclasses.dataclass class ServiceTopology: + images: typing.Optional[typing.Mapping] = dataclasses.field( + default_factory=topology_helpers.get_images + ) deployment: typing.Optional[objects.Deployment] = dataclasses.field( default_factory=topology_helpers.get_deployment ) @@ -23,7 +26,6 @@ class ServiceTopology: default_factory=topology_helpers.get_ingress ) - class SequencerDev(ServiceTopology): pass diff --git a/deployments/sequencer/services/topology_helpers.py b/deployments/sequencer/services/topology_helpers.py index 5bd5b74ead..ebab274b34 100644 --- a/deployments/sequencer/services/topology_helpers.py +++ b/deployments/sequencer/services/topology_helpers.py @@ -1,7 +1,20 @@ -from services import objects, const, helpers +from services import objects, const from config.sequencer import SequencerDevConfig +# replicas +# types +# config +# validators + +cluster_name = "gcp-integration" +replicas = 1 + +def get_images(): + return { + "sequencer": "us.gcr.io/starkware-dev/sequencer-node-test:0.0.1-dev.11" + } + def get_pvc() -> objects.PersistentVolumeClaim: return objects.PersistentVolumeClaim( access_modes=["ReadWriteOnce"], @@ -15,7 +28,7 @@ def get_pvc() -> objects.PersistentVolumeClaim: def get_dev_config(config_file_path: str) -> objects.Config: return SequencerDevConfig( - mount_path="/config/sequencer/presets/", config_file_path=config_file_path + config_file_path=config_file_path ) @@ -34,7 +47,7 @@ def get_ingress(url: str = "test.gcp-integration.sw-dev.io") -> objects.Ingress: host=url, paths=[ objects.IngressRuleHttpPath( - path="/monitoring/", + path="/monitoring", path_type="Prefix", backend_service_name="sequencer-node-service", backend_service_port_number=const.MONITORING_SERVICE_PORT, @@ -58,7 +71,7 @@ def get_service() -> objects.Service: ), objects.PortMapping( name="rpc", - port=const.RPC_SERVICE_PORT, + port=const.GRPC_SERVICE_PORT, container_port=const.RPC_CONTAINER_PORT, ), objects.PortMapping( @@ -77,8 +90,8 @@ def get_deployment() -> objects.Deployment: containers=[ objects.Container( name="server", - image="us.gcr.io/starkware-dev/sequencer-node-test:0.0.1-dev.3", - args=["--config_file", "/config/sequencer/presets/config"], + image="us.gcr.io/starkware-dev/sequencer-node-test:0.0.1-dev.8", + args=["--config_file", "/app/config/sequencer/presets/config"], ports=[ objects.ContainerPort(container_port=const.HTTP_CONTAINER_PORT), objects.ContainerPort(container_port=const.RPC_CONTAINER_PORT), @@ -108,7 +121,7 @@ def get_deployment() -> objects.Deployment: volume_mounts=[ objects.VolumeMount( name="config", - mount_path="/config/sequencer/presets/", + mount_path="/app/config/sequencer/presets/", read_only=True, ), objects.VolumeMount(name="data", mount_path="/data", read_only=False),