DO-2466 / rename to RADIXDLT_CORE_API_PORT (#122)

* rename to RADIXDLT_CORE_API_PORT * use system and core ports independently * make ports configurable * escape variables with double quotes in ci.yml * reformat code * reformat code * rename engine state api port
radixdlt · May 15, 2024 · 83a35f2 · 83a35f2
1 parent d85f1e4
commit 83a35f2
Show file tree

Hide file tree

Showing 19 changed files with 234 additions and 201 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -46,16 +46,16 @@ jobs:
         run: |
           cd node-runner-cli/
           pipenv install --python /usr/local/bin/python
-          snyk auth ${{ env.SNYK_TOKEN }}
-          snyk test --file=Pipfile --org=${{ env.SNYK_NETWORK_ORG_ID }} --severity-threshold=critical
+          snyk auth "${{ env.SNYK_TOKEN }}"
+          snyk test --file=Pipfile --org="${{ env.SNYK_NETWORK_ORG_ID }}" --severity-threshold=critical
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Run Snyk to check for code vulnerabilities
-        run: snyk code test --file=./node-runner-cli/Pipfile --org=${{ env.SNYK_NETWORK_ORG_ID }} --severity-threshold=high
+        run: snyk code test --file=./node-runner-cli/Pipfile --org="${{ env.SNYK_NETWORK_ORG_ID }}" --severity-threshold=high
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Test SBOM generation
-        run: snyk sbom --file=./node-runner-cli/Pipfile --org=${{ env.SNYK_NETWORK_ORG_ID }} --format=cyclonedx1.4+json > sbom.json
+        run: snyk sbom --file=./node-runner-cli/Pipfile --org="${{ env.SNYK_NETWORK_ORG_ID }}" --format=cyclonedx1.4+json > sbom.json
 
   snyk-monitor:
     runs-on: ubuntu-latest
@@ -95,11 +95,11 @@ jobs:
         run: |
           npm install snyk -g
           snyk -v
-          snyk auth ${{ env.SNYK_TOKEN }}
+          snyk auth "${{ env.SNYK_TOKEN }}"
       - name: Enable Snyk online monitoring - Devops
-        run: snyk monitor --file=./node-runner-cli/Pipfile --org=${{ env.SNYK_DEVOPS_ORG_ID }} --target-reference=${{ github.ref_name }}
+        run: snyk monitor --file=./node-runner-cli/Pipfile --org="${{ env.SNYK_DEVOPS_ORG_ID }}" --target-reference="${{ github.ref_name }}"
       - name: Enable Snyk online monitoring - Network
-        run: snyk monitor --file=./node-runner-cli/Pipfile --org=${{ env.SNYK_NETWORK_ORG_ID }} --target-reference=${{ github.ref_name }}
+        run: snyk monitor --file=./node-runner-cli/Pipfile --org="${{ env.SNYK_NETWORK_ORG_ID }}" --target-reference="${{ github.ref_name }}"
 
   upload-sbom:
     runs-on: ubuntu-latest
@@ -134,9 +134,9 @@ jobs:
         run: |
           npm install snyk -g
           snyk -v
-          snyk auth ${{ env.SNYK_TOKEN }}
+          snyk auth "${{ env.SNYK_TOKEN }}"
       - name: Generate SBOM
-        run: snyk sbom --file=./node-runner-cli/Pipfile --org=${{ env.SNYK_NETWORK_ORG_ID }} --format=cyclonedx1.4+json > sbom.json
+        run: snyk sbom --file=./node-runner-cli/Pipfile --org="${{ env.SNYK_NETWORK_ORG_ID }}" --format=cyclonedx1.4+json > sbom.json
       - name: Upload SBOM
         uses: RDXWorks-actions/action-gh-release@master
         with:
@@ -339,8 +339,8 @@ jobs:
           mkdir -p "$HOME/node-config"
           export PROMPT_FEEDS="node-runner-cli/test-prompts/core-gateway-all-local.yml"
           ./babylonnode docker config -m DETAILED \
-           -d $HOME/node-config \
-           -k $KEYSTORE_PASSWORD -nk -a
+           -d "$HOME/node-config" \
+           -k "$KEYSTORE_PASSWORD" -nk -a
         env:
           KEYSTORE_PASSWORD: ${{secrets.KEYSTORE_PASSWORD}}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/node-runner-cli/Pipfile.lock b/node-runner-cli/Pipfile.lock
diff --git a/node-runner-cli/api/CustomAPIClient.py b/node-runner-cli/api/CustomAPIClient.py
@@ -19,9 +19,9 @@ def set_default_header(self, header_name, header_value):
         self.default_headers[header_name] = header_value
 
     def prepare(
-        self,
-        http_method,
-        http_path,
+            self,
+            http_method,
+            http_path,
     ):
         req = requests.Request(
             http_method, f"{self._base_path}{http_path}", headers=self.default_headers

diff --git a/node-runner-cli/config/BaseConfig.py b/node-runner-cli/config/BaseConfig.py
@@ -52,8 +52,8 @@ def to_dict(self):
         for attr, value in class_variables.items():
             returning_dict[attr] = ""
             if (
-                type(self.__getattribute__(attr)) not in (str, int, bool, dict)
-                and self.__getattribute__(attr) is not None
+                    type(self.__getattribute__(attr)) not in (str, int, bool, dict)
+                    and self.__getattribute__(attr) is not None
             ):
                 returning_dict[attr] = self.__getattribute__(attr).to_dict()
             else:

diff --git a/node-runner-cli/config/CommonSystemDConfig.py b/node-runner-cli/config/CommonSystemDConfig.py
@@ -1,8 +1,6 @@
 import json
-import os
 
 from config.BaseConfig import BaseConfig, SetupMode
-from config.EnvVars import NGINX_BINARY_OVERIDE
 from config.Nginx import SystemdNginxConfig
 from github import github
 from utils.Network import Network

diff --git a/node-runner-cli/config/CoreDockerConfig.py b/node-runner-cli/config/CoreDockerConfig.py
@@ -23,7 +23,9 @@ def __init__(self, config_dict: dict):
         self.memory_limit: str = "14000m"
         self.validator_address: str = ""
         self.engine_state_enabled: bool = False
-        self.engine_state_port: str = "3336"
+        self.core_api_port: str = "3333"
+        self.system_api_port: str = "3334"
+        self.engine_state_api_port: str = "3336"
         self.engine_state_address: str = "0.0.0.0"
         self.java_opts: str = (
             "--enable-preview -server -Xms12g -Xmx12g  "

diff --git a/node-runner-cli/config/CoreSystemDConfig.py b/node-runner-cli/config/CoreSystemDConfig.py
@@ -25,7 +25,9 @@ def __init__(self, config_dict: dict):
         self.node_secrets_dir: str = "/etc/radixdlt/node/secrets"
         self.validator_address: str = ""
         self.engine_state_enabled: bool = False
-        self.engine_state_port: str = "3336"
+        self.core_api_port: str = "3333"
+        self.system_api_port: str = "3334"
+        self.engine_state_api_port: str = "3336"
         self.engine_state_address: str = "0.0.0.0"
         self.java_opts: str = (
             "--enable-preview -server -Xms12g -Xmx12g  "

diff --git a/node-runner-cli/config/Genesis.py b/node-runner-cli/config/Genesis.py
@@ -15,7 +15,7 @@ def create_genesis_file(genesis_json_location: str, genesis: str):
 
     @staticmethod
     def copy_genesis_file(
-        genesis_bin_data_file: str, genesis_files="testnet-genesis"
+            genesis_bin_data_file: str, genesis_files="testnet-genesis"
     ) -> str:
         bundle_dir = getattr(sys, "_MEIPASS", os.getcwd())
         path_to_genesis_bin_file = getenv(

diff --git a/node-runner-cli/config/Renderer.py b/node-runner-cli/config/Renderer.py
@@ -1,5 +1,3 @@
-import os
-import sys
 from os.path import dirname, join
 
 import yaml
@@ -8,7 +6,7 @@
 
 class Renderer:
     def load_file_based_template(
-        self, template_file_name: str, templates_dir="templates"
+            self, template_file_name: str, templates_dir="templates"
     ):
         templates_path = join(dirname(dirname(__file__)), templates_dir)
         self.env = Environment(

diff --git a/node-runner-cli/config/SystemDConfig.py b/node-runner-cli/config/SystemDConfig.py
@@ -111,7 +111,7 @@ def append_advanced_user_config(self, config_file: str, advanced_config_file: st
         f1.write(f2.read())
 
     def create_service_file(
-        self, service_file_path="/etc/systemd/system/radixdlt-node.service"
+            self, service_file_path="/etc/systemd/system/radixdlt-node.service"
     ):
         # This may need to be moved to jinja template
         tmp_service: str = "/tmp/radixdlt-node.service"

diff --git a/node-runner-cli/templates/radix-fullnode-compose.yml.j2 b/node-runner-cli/templates/radix-fullnode-compose.yml.j2
@@ -15,7 +15,8 @@ services:
     {% endif %}
     environment:
       JAVA_OPTS: {{core_node.java_opts or '--enable-preview -server -Xms12g -Xmx12g  -XX:MaxDirectMemorySize=2048m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseCompressedOops -Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts -Djavax.net.ssl.trustStoreType=jks -Djava.security.egd=file:/dev/urandom -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector'}}
-      RADIXDLT_API_PORT: 3333
+      RADIXDLT_CORE_API_PORT: {{core_node.core_api_port}}
+      RADIXDLT_SYSTEM_API_PORT: {{core_node.system_api_port}}
       RADIXDLT_LOG_LEVEL: {{core_node.log_level or 'info'}}
       RADIXDLT_NETWORK_ID: {{common_config.network_id}}
       RADIXDLT_NETWORK_SEEDS_REMOTE: {{core_node.trusted_node}}
@@ -38,7 +39,7 @@ services:
       {% if core_node.validator_address is defined %}
       RADIXDLT_CONSENSUS_VALIDATOR_ADDRESS: '{{core_node.validator_address}}'
       {% endif %}
-      RADIXDLT_ENGINE_STATE_API_PORT: {{core_node.engine_state_port}}
+      RADIXDLT_ENGINE_STATE_API_PORT: {{core_node.engine_state_api_port}}
       RADIXDLT_ENGINE_STATE_API_BIND_ADDRESS: {{core_node.engine_state_address}}
     image: {{core_node.repo}}:{{core_node.core_release}}
     init: true
@@ -47,6 +48,7 @@ services:
     {% if common_config.nginx_settings is not defined or (common_config.nginx_settings.protect_core | lower | trim) == "false" %}
     ports:
        - "3333:3333"
+       - "3334:3334"
        - "30000:30000"
     {% endif %}
     ulimits:
@@ -68,6 +70,8 @@ services:
     environment:
     {% if core_node is defined %}
       RADIXDLT_NETWORK_USE_PROXY_PROTOCOL: '{{core_node.use_proxy_protocol or 'false'}}'
+      RADIXDLT_CORE_API_PORT: '{{core_node.core_api_port}}'
+      RADIXDLT_SYSTEM_API_PORT: '{{core_node.system_api_port}}'
     {% endif %}
     {% if core_node is not defined or core_node is none %}
       RADIXDLT_ENABLE_TCP_CORE_PROXY: 'false'
@@ -76,7 +80,7 @@ services:
       RADIXDLT_GATEWAY_BEHIND_AUTH: '{{common_config.nginx_settings.gateway_behind_auth or 'false'}}'
     {% if core_node is not none and core_node is defined %}
       RADIXDLT_ENGINE_STATE_ENABLE: '{{core_node.engine_state_enabled or 'false'}}'
-      RADIXDLT_ENGINE_STATE_PORT: '{{core_node.engine_state_port}}'
+      RADIXDLT_engine_state_api_port: '{{core_node.engine_state_api_port}}'
     {% endif %}
     image: {{common_config.nginx_settings.repo}}:{{common_config.nginx_settings.release}}
     ports:

diff --git a/node-runner-cli/templates/systemd-default.config.j2 b/node-runner-cli/templates/systemd-default.config.j2
@@ -22,11 +22,14 @@ network.host_ip={{common_config.host_ip}}
 
 log.level=debug
 
-api.port=3334
+api.port={{core_node.core_api_port}}
 api.transactions.enable=true
 api.sign.enable=true
 api.bind.address=0.0.0.0
 
+api.system.port={{core_node.system_api_port}}
+api.system.bind_address=0.0.0.0
+
 db.location=/home/radixdlt/babylon-ledger
 
 {% if core_node.validator_address is defined and core_node.validator_address is not none %}
@@ -45,7 +48,7 @@ genesis.olympia.node_bech32_address={{migration.olympia_node_bech32_address}}
 
 
 {% if core_node.engine_state_enabled %}
-api.engine_state.port={{core_node.engine_state_port}}
+api.engine_state.port={{core_node.engine_state_api_port}}
 api.engine_state.bind_address={{core_node.engine_state_address}}
 db.re_node_listing_indices.enable={{core_node.engine_state_enabled}}
 db.historical_substate_values.enable={{core_node.engine_state_enabled}}

diff --git a/node-runner-cli/tests/fixtures/docker-compose.yaml b/node-runner-cli/tests/fixtures/docker-compose.yaml
@@ -7,7 +7,8 @@ services:
       JAVA_OPTS: --enable-preview -server -Xms12g -Xmx12g  -XX:MaxDirectMemorySize=2048m
         -XX:+HeapDumpOnOutOfMemoryError -XX:+UseCompressedOops -Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts
         -Djavax.net.ssl.trustStoreType=jks -Djava.security.egd=file:/dev/urandom -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
-      RADIXDLT_API_PORT: 3333
+      RADIXDLT_CORE_API_PORT: 3333
+      RADIXDLT_SYSTEM_API_PORT: 3334
       RADIXDLT_GENESIS_DATA_FILE: /home/radixdlt/genesis_data_file.bin
       RADIXDLT_LOG_LEVEL: info
       RADIXDLT_NETWORK_ID: 33

diff --git a/node-runner-cli/tests/unit/test_docker.py b/node-runner-cli/tests/unit/test_docker.py
@@ -15,24 +15,24 @@ class DockerUnitTests(unittest.TestCase):
     def test_docker_config(self, mockout):
         urllib3.disable_warnings()
         with patch(
-            "builtins.input",
-            side_effect=["N", "N", "/home/runner/docker-compose.yml", "N"],
+                "builtins.input",
+                side_effect=["N", "N", "/home/runner/docker-compose.yml", "N"],
         ):
             with patch(
-                "sys.argv",
-                [
-                    "main",
-                    "docker",
-                    "config",
-                    "-m",
-                    "DETAILED",
-                    "-n",
-                    "2",
-                    "-k",
-                    "radix",
-                    "-nk",
-                    "-a",
-                ],
+                    "sys.argv",
+                    [
+                        "main",
+                        "docker",
+                        "config",
+                        "-m",
+                        "DETAILED",
+                        "-n",
+                        "2",
+                        "-k",
+                        "radix",
+                        "-nk",
+                        "-a",
+                    ],
             ):
                 main()
 

diff --git a/node-runner-cli/tests/unit/test_gateway_setup.py b/node-runner-cli/tests/unit/test_gateway_setup.py
@@ -209,7 +209,7 @@ def test_setup_gateway_compose_file_fixture_test(self, mockout):
                 self.assertEqual(f1.read(), f2.read())
 
     def expect_ask_gateway_inputs_get_inserted_into_object(
-        self, config, questionary_keyboard_input
+            self, config, questionary_keyboard_input
     ):
         self.assertEqual(
             questionary_keyboard_input[0],

diff --git a/node-runner-cli/tests/unit/test_monitoring.py b/node-runner-cli/tests/unit/test_monitoring.py
@@ -67,24 +67,24 @@ def test_template_failure(self, mock_stdout):
     @mock.patch("sys.stdout", new_callable=StringIO)
     def test_monitoring_config(self, mock_out):
         with mock.patch(
-            "builtins.input",
-            side_effect=["Y", "https://45.152.180.182", "metrics", "testpassword", "n"],
+                "builtins.input",
+                side_effect=["Y", "https://45.152.180.182", "metrics", "testpassword", "n"],
         ):
             with mock.patch("sys.argv", ["main", "monitoring", "config"]):
                 main()
 
         with mock.patch(
-            "sys.argv",
-            ["main", "monitoring", "config", "-m", "MONITOR_CORE", "-cm", "test"],
+                "sys.argv",
+                ["main", "monitoring", "config", "-m", "MONITOR_CORE", "-cm", "test"],
         ):
             main()
 
         with mock.patch(
-            "builtins.input",
-            side_effect=["Y", "https://45.152.180.182", "metrics", "testpassword", "n"],
+                "builtins.input",
+                side_effect=["Y", "https://45.152.180.182", "metrics", "testpassword", "n"],
         ):
             with mock.patch(
-                "sys.argv", ["main", "monitoring", "config", "-m", "DETAILED"]
+                    "sys.argv", ["main", "monitoring", "config", "-m", "DETAILED"]
             ):
                 main()