feat: add resource configuration for prometheus and grafana (#773)

Please note: 
`grafana_additional_dashboards` ->
`grafana_params.additional_dashboards`

README.md

@@ -607,6 +607,30 @@ goomy_blob_params:
   # A list of optional params that will be passed to the blob-spammer comamnd for modifying its behaviour
   goomy_blob_args: []
 
+# Configuration place for prometheus
+prometheus_params:
+  storage_tsdb_retention_time: "1d"
+  storage_tsdb_retention_size: "512MB"
+  # Resource management for prometheus container
+  # CPU is milicores
+  # RAM is in MB
+  min_cpu: 10
+  max_cpu: 1000
+  min_mem: 128
+  max_mem: 2048
+
+# Configuration place for grafana
+grafana_params:
+  # A list of locators for grafana dashboards to be loaded be the grafana service
+  additional_dashboards: []
+  # Resource management for grafana container
+  # CPU is milicores
+  # RAM is in MB
+  min_cpu: 10
+  max_cpu: 1000
+  min_mem: 128
+  max_mem: 2048
+
 # Configuration place for the assertoor testing tool - https://github.com/ethpandaops/assertoor
 assertoor_params:
   # Assertoor docker image to use
@@ -695,9 +719,6 @@ parallel_keystore_generation: false
 # Default to false
 disable_peer_scoring: false
 
-# A list of locators for grafana dashboards to be loaded be the grafana service
-grafana_additional_dashboards: []
-
 # Whether the environment should be persistent; this is WIP and is slowly being rolled out accross services
 # Note this requires Kurtosis greater than 0.85.49 to work
 # Note Erigon, Besu, Teku persistence is not currently supported with docker.

main.star

@@ -638,8 +638,7 @@ def run(plan, args={}):
             all_ethereum_metrics_exporter_contexts,
             all_xatu_sentry_contexts,
             global_node_selectors,
-            args_with_right_defaults.prometheus_params.storage_tsdb_retention_time,
-            args_with_right_defaults.prometheus_params.storage_tsdb_retention_size,
+            args_with_right_defaults.prometheus_params,
         )
 
         plan.print("Launching grafana...")
@@ -649,7 +648,7 @@ def run(plan, args={}):
             grafana_dashboards_config_template,
             prometheus_private_url,
             global_node_selectors,
-            additional_dashboards=args_with_right_defaults.grafana_additional_dashboards,
+            args_with_right_defaults.grafana_params,
         )
         plan.print("Successfully launched grafana")
 

network_params.yaml

@@ -93,6 +93,19 @@ tx_spammer_params:
   tx_spammer_extra_args: []
 goomy_blob_params:
   goomy_blob_args: []
+prometheus_params:
+  storage_tsdb_retention_time: "1d"
+  storage_tsdb_retention_size: "512MB"
+  min_cpu: 10
+  max_cpu: 1000
+  min_mem: 128
+  max_mem: 2048
+grafana_params:
+  additional_dashboards: []
+  min_cpu: 10
+  max_cpu: 1000
+  min_mem: 128
+  max_mem: 2048
 assertoor_params:
   image: ""
   run_stability_check: false
@@ -108,7 +121,6 @@ snooper_enabled: false
 ethereum_metrics_exporter_enabled: false
 parallel_keystore_generation: false
 disable_peer_scoring: false
-grafana_additional_dashboards: []
 persistent: false
 mev_type: null
 mev_params:

src/grafana/grafana_launcher.star

@@ -38,20 +38,14 @@ USED_PORTS = {
     )
 }
 
-# The min/max CPU/memory that grafana can use
-MIN_CPU = 10
-MAX_CPU = 1000
-MIN_MEMORY = 128
-MAX_MEMORY = 2048
-
 
 def launch_grafana(
     plan,
     datasource_config_template,
     dashboard_providers_config_template,
     prometheus_private_url,
     global_node_selectors,
-    additional_dashboards=[],
+    grafana_params,
 ):
     (
         grafana_config_artifacts_uuid,
@@ -62,7 +56,7 @@ def launch_grafana(
         datasource_config_template,
         dashboard_providers_config_template,
         prometheus_private_url,
-        additional_dashboards=additional_dashboards,
+        additional_dashboards=grafana_params.additional_dashboards,
     )
 
     merged_dashboards_artifact_name = merge_dashboards_artifacts(
@@ -75,6 +69,7 @@ def launch_grafana(
         grafana_config_artifacts_uuid,
         merged_dashboards_artifact_name,
         global_node_selectors,
+        grafana_params,
     )
 
     plan.add_service(SERVICE_NAME, config)
@@ -130,6 +125,7 @@ def get_config(
     grafana_config_artifacts_name,
     grafana_dashboards_artifacts_name,
     node_selectors,
+    grafana_params,
 ):
     return ServiceConfig(
         image=IMAGE_NAME,
@@ -145,10 +141,10 @@ def get_config(
             GRAFANA_CONFIG_DIRPATH_ON_SERVICE: grafana_config_artifacts_name,
             GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE: grafana_dashboards_artifacts_name,
         },
-        min_cpu=MIN_CPU,
-        max_cpu=MAX_CPU,
-        min_memory=MIN_MEMORY,
-        max_memory=MAX_MEMORY,
+        min_cpu=grafana_params.min_cpu,
+        max_cpu=grafana_params.max_cpu,
+        min_memory=grafana_params.min_mem,
+        max_memory=grafana_params.max_mem,
         node_selectors=node_selectors,
     )
 

src/package_io/input_parser.star

@@ -95,11 +95,12 @@ def input_parser(plan, input_args):
         result["additional_services"] = DEFAULT_ADDITIONAL_SERVICES
     else:
         result["additional_services"] = []
-    result["grafana_additional_dashboards"] = []
     result["tx_spammer_params"] = get_default_tx_spammer_params()
     result["custom_flood_params"] = get_default_custom_flood_params()
     result["disable_peer_scoring"] = False
     result["goomy_blob_params"] = get_default_goomy_blob_params()
+    result["prometheus_params"] = get_default_prometheus_params()
+    result["grafana_params"] = get_default_grafana_params()
     result["assertoor_params"] = get_default_assertoor_params()
     result["prometheus_params"] = get_default_prometheus_params()
     result["xatu_sentry_params"] = get_default_xatu_sentry_params()
@@ -331,6 +332,17 @@ def input_parser(plan, input_args):
             storage_tsdb_retention_size=result["prometheus_params"][
                 "storage_tsdb_retention_size"
             ],
+            min_cpu=result["prometheus_params"]["min_cpu"],
+            max_cpu=result["prometheus_params"]["max_cpu"],
+            min_mem=result["prometheus_params"]["min_mem"],
+            max_mem=result["prometheus_params"]["max_mem"],
+        ),
+        grafana_params=struct(
+            additional_dashboards=result["grafana_params"]["additional_dashboards"],
+            min_cpu=result["grafana_params"]["min_cpu"],
+            max_cpu=result["grafana_params"]["max_cpu"],
+            min_mem=result["grafana_params"]["min_mem"],
+            max_mem=result["grafana_params"]["max_mem"],
         ),
         apache_port=result["apache_port"],
         assertoor_params=struct(
@@ -362,7 +374,6 @@ def input_parser(plan, input_args):
         ethereum_metrics_exporter_enabled=result["ethereum_metrics_exporter_enabled"],
         xatu_sentry_enabled=result["xatu_sentry_enabled"],
         parallel_keystore_generation=result["parallel_keystore_generation"],
-        grafana_additional_dashboards=result["grafana_additional_dashboards"],
         disable_peer_scoring=result["disable_peer_scoring"],
         persistent=result["persistent"],
         xatu_sentry_params=struct(
@@ -936,6 +947,10 @@ def get_default_mev_params(mev_type, preset):
         "labels": None,
         "storage_tsdb_retention_time": "1d",
         "storage_tsdb_retention_size": "512MB",
+        "min_cpu": 10,
+        "max_cpu": 1000,
+        "min_mem": 128,
+        "max_mem": 2048,
     }
 
     if mev_type == constants.MEV_RS_MEV_TYPE:
@@ -997,6 +1012,20 @@ def get_default_prometheus_params():
     return {
         "storage_tsdb_retention_time": "1d",
         "storage_tsdb_retention_size": "512MB",
+        "min_cpu": 10,
+        "max_cpu": 1000,
+        "min_mem": 128,
+        "max_mem": 2048,
+    }
+
+
+def get_default_grafana_params():
+    return {
+        "additional_dashboards": [],
+        "min_cpu": 10,
+        "max_cpu": 1000,
+        "min_mem": 128,
+        "max_mem": 2048,
     }
 
 

src/package_io/sanity_check.star

@@ -154,6 +154,21 @@ SUBCATEGORY_PARAMS = {
     "goomy_blob_params": [
         "goomy_blob_args",
     ],
+    "prometheus_params": [
+        "min_cpu",
+        "max_cpu",
+        "min_mem",
+        "max_mem",
+        "storage_tsdb_retention_time",
+        "storage_tsdb_retention_size",
+    ],
+    "grafana_params": [
+        "additional_dashboards",
+        "min_cpu",
+        "max_cpu",
+        "min_mem",
+        "max_mem",
+    ],
     "assertoor_params": [
         "image",
         "run_stability_check",
@@ -164,10 +179,6 @@ SUBCATEGORY_PARAMS = {
         "run_lifecycle_test",
         "tests",
     ],
-    "prometheus_params": [
-        "storage_tsdb_retention_time",
-        "storage_tsdb_retention_size",
-    ],
     "mev_params": [
         "mev_relay_image",
         "mev_builder_image",
@@ -228,7 +239,6 @@ ADDITIONAL_CATEGORY_PARAMS = {
     "ethereum_metrics_exporter_enabled": "",
     "parallel_keystore_generation": "",
     "disable_peer_scoring": "",
-    "grafana_additional_dashboards": "",
     "persistent": "",
     "mev_type": "",
     "xatu_sentry_enabled": "",

src/prometheus/prometheus_launcher.star

@@ -12,12 +12,6 @@ METRICS_INFO_ADDITIONAL_CONFIG_KEY = "config"
 
 PROMETHEUS_DEFAULT_SCRAPE_INTERVAL = "15s"
 
-# The min/max CPU/memory that prometheus can use
-MIN_CPU = 10
-MAX_CPU = 1000
-MIN_MEMORY = 128
-MAX_MEMORY = 2048
-
 
 def launch_prometheus(
     plan,
@@ -28,8 +22,7 @@ def launch_prometheus(
     ethereum_metrics_exporter_contexts,
     xatu_sentry_contexts,
     global_node_selectors,
-    storage_tsdb_retention_time,
-    storage_tsdb_retention_size,
+    prometheus_params,
 ):
     metrics_jobs = get_metrics_jobs(
         el_contexts,
@@ -43,13 +36,13 @@ def launch_prometheus(
         plan,
         metrics_jobs,
         "prometheus",
-        MIN_CPU,
-        MAX_CPU,
-        MIN_MEMORY,
-        MAX_MEMORY,
+        min_cpu=prometheus_params.min_cpu,
+        max_cpu=prometheus_params.max_cpu,
+        min_memory=prometheus_params.min_mem,
+        max_memory=prometheus_params.max_mem,
         node_selectors=global_node_selectors,
-        storage_tsdb_retention_time=storage_tsdb_retention_time,
-        storage_tsdb_retention_size=storage_tsdb_retention_size,
+        storage_tsdb_retention_time=prometheus_params.storage_tsdb_retention_time,
+        storage_tsdb_retention_size=prometheus_params.storage_tsdb_retention_size,
     )
 
     return prometheus_url