diff --git a/deployment/dda/dda.yaml b/deployment/dda/dda.yaml new file mode 100644 index 00000000..fd08c1fe --- /dev/null +++ b/deployment/dda/dda.yaml @@ -0,0 +1,374 @@ +# SPDX-FileCopyrightText: © 2024 Siemens AG +# SPDX-License-Identifier: MIT + +# Data Distribution Agent (DDA) Configuration +# https://github.com/coatyio/dda + +# DDA configuration version number as a string. +# +# The version number must conform to the YAML configuration schema version used +# within the DDA module. +version: "1" + +# Identity of a DDA. +identity: + # Name of agent. + # + # If not present, default name is "DDA". + name: edgeless-dda-sidecar + + # Unique identifier of agent as a string. + # + # You may use a UUID v4 string to make identifiers globally unique among all + # Data Distribution Agents within your system. + # + # If not present, default id is a UUID v4 newly generated at startup of the + # DDA application. + id: + +# Identifies the application context in which the agent operates. Only agents +# within the same cluster can interact with each other in a DDA communication +# network. Multiple clusters can be run isolated from each other and in parallel +# in the same DDA communication network. +# +# Must be a string consisting of lower-case ASCII letters ('a' to 'z'), +# upper-case ASCII letters ('A' to 'Z'), ASCII digits ('0' to '9'), ASCII dot +# ('.'), ASCII hyphen (-), or ASCII underscore (_). +# +# If not present, default cluster name is "dda". +cluster: + +# Server-side configuration of public peripheral DDA Client APIs. +apis: + # DDA gRPC service. + grpc: + # gRPC server address of gRPC API. When run inside Docker the port of the + # given address must be exposed/mapped to the host system. + # + # The DDA gRPC server supports connections over TCP or Unix domain sockets. + # Unix sockets provide fast, reliable, stable and low latency communication + # between gRPC client and server co-located on the _same_ machine. Note that + # a DDA sidecar hosted in a local container needs to share the configured + # socket file via a volume. + # + # For TCP, the address string has the form "host:port". The port must be a + # literal port number or a service name. If the host is empty or a literal + # unspecified IP address, as in ":80", "0.0.0.0:80" or "[::]:80", the local + # system is assumed. + # + # For Unix sockets, the address string has the form "unix:socketfile", e.g. + # "unix:/tmp/mydda.sock". + # + # If not present, default address string is ":8900". + address: + + # Indicates whether this API service should be disabled within your DDA + # sidecar or instance. Specify true to opt out from exposing this API + # service. This is useful if you embed the DDA as a library in your client + # application. + # + # If not present, defaults to false. + disabled: false + + # After a duration of this time if the server doesn't see any activity it + # pings the client to see if the transport is still alive. If set below 1s, + # a minimum value of 1s will be used instead. + # + # If not present, defaults to 2 hours ("2h"). + # + # Configuring gRPC HTTP/2 keepalives can be useful in a variety of + # situations, including when sending data over a long-lived connection which + # might be considered as idle by proxy or load balancers, when the network + # is less reliable (e.g. mobile applications), when using a connection after + # a long period of inactivity. Note that gRPC client-side and server-side + # keepalives must be configured in a compatible way. + # + # A duration string is a sequence of decimal numbers, each with optional + # fraction and a unit suffix, such as "300ms", "1.5h" or "2h45m". Valid time + # units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + keepalive: + + # DDA gRPC-Web service. + grpc-web: + # gRPC-Web server address of gRPC-Web API. When run inside Docker the port + # of the given address must be exposed/mapped to the host system. + # + # The address string has the form "host:port". The port must be a literal + # port number or a service name. If the host is empty or a literal + # unspecified IP address, as in ":80", "0.0.0.0:80" or "[::]:80", the local + # system is assumed. + # + # If not present, default address string is ":8800". + address: + + # Valid origins in the form of a Yaml array list for which responses can be + # shared with requesting code (allowed CORS origin requests). + # + # If not present or an empty list, all requests from all origins are + # allowed. + # + # For example, to only allow requests from two specific origins: + # [https://example.org,https://awesome.com:4200] + access-control-allow-origin: + + # Indicates whether this API service should be disabled within your DDA + # sidecar or instance. Specify true to opt out from exposing this API + # service. This is useful if you embed the DDA as a library in your client + # application. + # + # Note that an enabled gRPC-Web API is only usable and reachable if the gRPC + # API is also enabled. + # + # If not present, defaults to false. + disabled: false + + # Certificate file in PEM format used for TLS API server authentication. + # + # A relative path refers to the current working directory where the + # (dockerized) dda program is started. + # + # If not present, defaults to "" (i.e. no TLS). If this option is set, option + # key must also be set to enable TLS authentication, and vice versa. + cert: + + # Private key file in PEM format used for TLS API server authentication. + # + # A relative path refers to the current working directory where the + # (dockerized) dda program is started. + # + # If not present, defaults to "" (i.e. no TLS). If this option is set, option + # cert must also be set to enable TLS authentication, and vice versa. + key: + +# Configuration of peripheral DDA services. +services: + # Communication service. + com: + # The pub-sub messaging protocol to use. One of: "mqtt5" (MQTT v5) + # + # If not present, "mqtt5" is used by default. + protocol: + + # Server URL for broker-based pub-sub messaging protocols. + # + # The supported URL schemes are protocol-specific: + # + # - mqtt5: mqtt, tcp (plain TCP), mqtts, tls (TLS, SSL), ws (WebSockets), + # wss (secure WebSockets) + # + # If not present, a protocol-specific default is used: + # + # - mqtt5: tcp://localhost:1883 + url: tcp://mqtt_broker:1883 + + # Authentication options. + auth: + # Authentication method. + # + # One of: + # + # - none: connection is unauthenticated + # - tls: secure TLS connection with client certificate and private key + # + # The other authentication options are method-specific. The ones which do + # not relate to the specified method are ignored. + # + # If not present or set to "none", no authentication method is used. + method: + + # Client Certificate file in PEM format used for TLS authentication (for + # auth method tls only). + # + # A relative path refers to the current working directory where the + # (dockerized) dda program is started. + # + # If not present, defaults to "". If this option is set, option key must + # also be set, and vice versa. + cert: + + # Client Private key file in PEM format used for TLS authentication (for + # auth method tls only). + # + # A relative path refers to the current working directory where the + # (dockerized) dda program is started. + # + # If not present, defaults to "". If this option is set, option cert must + # also be set, and vice versa. + key: + + # Whether to verify the server certificate against the list of supplied + # Certificate Authorities and reject if verification fails (for auth + # method tls only). + # + # WARNING: Never set this option to false in production environments as + # you are exposing yourself to man in the middle attacks. + # + # If not present, defaults to true. + verify: + + # Username (may be used with all auth methods). + # + # Use with auth method none to enable basic authentication with username + # and password. + # + # May be used with the following protocols: mqtt5 + # + # If not present, defaults to "". + username: + + # Password or other credential (may be used with all auth methods). + # + # Use with auth method none to enable basic authentication with username + # and password. + # + # If a credential other than a password is given, e.g. an authentication + # token, the field username may be left empty. + # + # May be used with the following protocols: mqtt5 + # + # If not present, defaults to "". + password: + + # Options related to the underlying protocol implementation or communication + # network in the form of a Yaml object with dynamic key-value pairs. + # + # Supported options for protocol "mqtt5": + # debug - if true, enables debug logging for MQTT library (defaults to false) + # qos - QoS used for all publications and subscriptions, one of 0, 1, 2 (defaults to 0) + # strictClientId - if true, generate strict MQTT client ID (defaults to false) + # keepAlive - keep alive interval in seconds (defaults to 30) + # connectRetryDelay - interval in millis between reconnection attempts (defaults to 1000) + # connectTimeout - millis to wait for the connection process to complete (defaults to 10000) + # noLocal - if true, publications are not forwarded to the originating MQTT client (defaults to false) + opts: + + # Indicates whether this service should be disabled within your DDA sidecar + # or instance. Specify true to opt out from exposing this service. + # + # If not present, defaults to false. + disabled: false + + # Local persistent or in-memory key-value storage service for a single DDA + # sidecar or instance. + store: + # The underlying storage engine to use. One of: "pebble". + # + # - pebble: a performance-optimized embedded storage engine based on LSM + # with WAL. Storage is represented either as a directory with multiple + # files on disk or as a non-persistent in-memory store. + # + # If not present, "pebble" is used by default. + engine: + + # Location where data is stored on a local file system. In a containerized + # DDA sidecar use a volume or a bind mount and specify a destination path + # that corresponds with this storage location. + # + # The specified location is specific to the configured storage engine: + # + # - pebble: a directory given by an absolute pathname or a pathname relative + # to the working directory of the DDA sidecar or instance, or an empty + # string to indicate that the store is non-persistent and completely + # memory-backed as long as the DDA sidecar is running. If the specified + # storage directory doesn't exist, it and its parent directories are + # created by the storage service. + # + # If not present, a storage engine specific default is used: + # + # - pebble: "" (in-memory storage) + location: + + # Indicates whether this service should be disabled within your DDA sidecar + # or instance. Specify true to opt out from exposing this service. + # + # If not present, defaults to true. + disabled: false + + # State synchronization service based on a consensus protocol, like Raft or + # Paxos, that guarantees strong consistency on a replicated state. + # + # Configuration options are used to set up a local member belonging to a state + # synchronization group formed by the subset of agents in the configured DDA + # cluster which have enabled this service. All members within the same DDA + # cluster share replicated state. State synchronization groups in different + # clusters run isolated from each other and in parallel to provide + # cluster-specific replicated state. + # + # Note that each member of a state synchronization group is assigned a unique + # member ID as configured by the DDA identity id. To efficiently restore + # persisted member state after a DDA restart, this unique id should not change + # between restarts. + state: + # The consensus protocol to use. One of: "raft" + # + # If not present, "raft" is used by default. + protocol: + + # Persistent storage location on a local file system used to store + # replicated state, snapshots, and log entries of the local state + # synchronization group member. In a containerized DDA sidecar use a volume + # or a bind mount and specify a destination path that corresponds with this + # storage location. + # + # Specify a directory given by an absolute pathname or a pathname relative + # to the working directory of the DDA sidecar or instance. + # + # You may also specify an empty string to indicate that the store is + # non-persistent and completely memory-backed as long as the DDA agent is + # running. Usually, you shouldn't use non-persistent storage in a production + # environment as all local state is lost when shutting down the DDA sidecar. + # Non-persistent storage is mainly used for testing purposes. + # + # If not present or empty, in-memory non-persistent storage is used. + store: + + # Indicates whether the local member should initially create the state + # synchronization group within the configured DDA cluster. Specify true for + # exactly one member within the cluster; specify false for all other members + # within the cluster. Multiple members within the same cluster having this + # option set to true will lead to undefined behavior. + # + # Note that the member that creates the state synchronization group is not + # required to stay in the group. It is possible to use one designated member + # for creating the group that leaves the group after it is up and running. + # In this case it is important that at least one other member has joined the + # group before the creating member leaves. + # + # Also note that the member that creates the state synchronization group is + # required to be the first DDA instance to be started up. All other members + # must be started up after the bootstrapping member has been set up. Not + # respecting this requirement will lead to undefined behavior and may result + # in partitioned members that form their own group. + # + # Note that if you restart a bootstrapping member after a graceful or + # non-graceful shutdown the bootstrap flag should be reconfigured with false + # as the still existing group will have elected a different leading member + # in the meantime. + # + # If not present, defaults to false. + bootstrap: true + + # Indicates whether this service should be disabled within your DDA sidecar + # or instance. Specify true to opt out from exposing this service. + # + # Note that if this service is enabled the com service must also be enabled + # since messaging between state synchronization group members makes use of + # the DDA communication service internally. + # + # If not present, defaults to true. + disabled: false + + # Options related to the underlying consensus protocol implementation in the + # form of a Yaml object with dynamic key-value pairs. + # + # Supported options for protocol "raft": + # debug - if true, enables debug logging for Raft library (defaults to false) + # startupTimeout - millis to wait for startup of a new Raft node to complete (defaults to 10000) + # lfwTimeout - millis to wait for a leader forwarded Propose/GetState response (defaults to 20000) + # rpcTimeout - millis to wait for a remote operation on Raft transport to complete (defaults to 1000) + # heartbeatTimeout - millis to wait in follower state before attempting an election (defaults to 1000) + # electionTimeout - millis to wait in candidate state before attempting an election (defaults to 1000) + # installSnapshotTimeoutScale - timeout scale factor in bytes for snapshot installations (defaults to 256*1024) + # snapshotInterval - millis to check how often a snapshot should be performed (defaults to 120000) + # snapshotThreshold - minimum number of outstanding logs to start snapshotting (defaults to 8192) + opts: diff --git a/deployment/node/start.sh b/deployment/node/start.sh index 1056c7ad..6a912c40 100644 --- a/deployment/node/start.sh +++ b/deployment/node/start.sh @@ -26,11 +26,29 @@ num_cpus = 1 clock_freq_cpu = 1000 num_cores = $NUM_CORES EOF + elif [ $"$NODE_TYPE" == "FILE_LOG" ] ; then cat >> node.toml << EOF [resources] file_log_provider = "file-log-1" EOF + +# DDA type activates the dda resource provider and wasm runtime +elif [ $"$NODE_TYPE" == "DDA" ] ; then + cat >> node.toml << EOF +[wasm_runtime] +enabled = true + +[user_node_capabilities] +labels = $LABELS +num_cpus = 1 +clock_freq_cpu = 1000 +num_cores = $NUM_CORES + +[resources] +dda_provider = "dda-1" +EOF + else echo "invalid NODE_TYPE: $NODE_TYPE" exit 1 diff --git a/edgeless_inabox/src/bin/edgeless_inabox.rs b/edgeless_inabox/src/bin/edgeless_inabox.rs index 4cdf1b37..5093b44d 100644 --- a/edgeless_inabox/src/bin/edgeless_inabox.rs +++ b/edgeless_inabox/src/bin/edgeless_inabox.rs @@ -187,10 +187,6 @@ fn generate_configs(number_of_nodes: i32) -> Result { true => Some("redis-1".to_string()), false => None, }, - dda_url: match first_node { - true => Some(next_url()), - false => None, - }, dda_provider: match first_node { true => Some("dda-1".to_string()), false => None, diff --git a/edgeless_node/src/lib.rs b/edgeless_node/src/lib.rs index 6b1e4397..adfcad87 100644 --- a/edgeless_node/src/lib.rs +++ b/edgeless_node/src/lib.rs @@ -97,8 +97,6 @@ pub struct EdgelessNodeResourceSettings { /// value of a given given, as specified in the resource configuration /// at run-time. pub redis_provider: Option, - /// The URL of DDA used by this node, used for communication via the DDA resources - pub dda_url: Option, /// If not empty, a DDA resource with that name is created. pub dda_provider: Option, /// The ollama resource provider settings. @@ -367,9 +365,9 @@ async fn fill_resources( } } - if let (Some(dda_url), Some(provider_id)) = (&settings.dda_url, &settings.dda_provider) { - if !dda_url.is_empty() && !provider_id.is_empty() { - log::info!("Creating resource '{}' at {}", provider_id, dda_url); + if let Some(provider_id) = &settings.dda_provider { + if !provider_id.is_empty() { + log::info!("Creating dda resource provider '{}'", provider_id); let class_type = "dda".to_string(); ret.insert( provider_id.clone(), @@ -720,7 +718,6 @@ http_ingress_provider = "http-ingress-1" http_egress_provider = "http-egress-1" file_log_provider = "file-log-1" redis_provider = "redis-1" -dda_url = "http://127.0.0.1:10000" dda_provider = "dda-1" kafka_egress_provider = "kafka-egress-1" diff --git a/edgeless_systemtests/src/lib.rs b/edgeless_systemtests/src/lib.rs index 00840c69..b3213d1c 100644 --- a/edgeless_systemtests/src/lib.rs +++ b/edgeless_systemtests/src/lib.rs @@ -107,7 +107,6 @@ mod tests { http_egress_provider: None, file_log_provider: Some("file-log-1".to_string()), redis_provider: None, - dda_url: None, dda_provider: None, ollama_provider: None, kafka_egress_provider: None,