From 2388fd009d6bc240b2fc9366c53a8cb8fe11426a Mon Sep 17 00:00:00 2001 From: Naorem Khogendro Singh Date: Fri, 17 Nov 2023 16:32:08 -0800 Subject: [PATCH] [PLAT-11473] Node agent fails to come up on air-gap public cloud providers like aws due to selinux Summary: In airgap installation, skip installing semanage and use chcon (available) instead. Also move the firewall exception addition outside of selinux check. Test Plan: 1. Created a provider with airgap enabled. 2. Create a universe. 3. Verified the command line contains --airgap. ``` 2023-11-18T00:30:31.486Z [info] 5ad4fc3d-d0c1-4cb9-b906-b25818a5ddc8 ShellProcessHandler.java:185 [TaskPool-CreateUniverse(fc31acad-a358-4622-8345-ada324fbb0cc)-0] com.yugabyte.yw.common.ShellProcessHandler Starting proc (full cmd) - 'bin/py_wrapper' 'bin/run_node_action.py' '--is_master' '--node_name' 'yb-admin-nsingh-test-universe1-n1' 'ssh' '--port' '22' '--ip' '10.9.117.17' '--key' '/opt/yugaware/keys/ffc283e8-eed6-4cdb-a4c5-a7370866cd17/yb-admin-aws-airgap1_ffc283e8-eed6-4cdb-a4c5-a7370866cd17-key.pem' '--user' 'ec2-user' 'run_command' '--command' 'sudo' '-H' '/bin/bash' '-c' 'rm -rf /root/node-agent && tar -zxf /tmp/node-agent-1700267413499/node-agent/release/node-agent.tgz --strip-components=3 -C /tmp/node-agent-1700267413499/node-agent --wildcards */node-agent-installer.sh && mv -f /tmp/node-agent-1700267413499/node-agent /root/node-agent && chmod +x /root/node-agent/node-agent-installer.sh && /root/node-agent/node-agent-installer.sh -c install --skip_verify_cert --disable_egress --id 23e1d6cd-26c5-4fbc-b5ac-0f27ad5f2cf2 --customer_id f33e3c9b-75ab-4c30-80ad-cba85646ea39 --cert_dir 25c6d086-e242-452a-bdb3-15bce302a5f0 --node_name yb-admin-nsingh-test-universe1-n1 --node_ip 10.9.117.17 --node_port 10070 --airgap && chmod 755 /root /root/node-agent && rm -rf /root/node-agent/node-agent-installer.sh' - logging stdout=/tmp/shell_process_out13128525809232153353tmp, stderr=/tmp/shell_process_err6772678925750908315tmp ``` 3. Universe creation succeeded. 4. Verified that semanage was not installed. ``` [ec2-user@ip-10-9-117-17 ~]$ sudo semamage sudo: semamage: command not found ``` Reviewers: cwang, nbhatia, sanketh Reviewed By: cwang Subscribers: yugaware Differential Revision: https://phorge.dev.yugabyte.com/D30331 --- .../resources/node-agent-installer.sh | 44 +++++++++++++------ .../commissioner/tasks/UniverseTaskBase.java | 10 +++++ .../tasks/subtasks/InstallNodeAgent.java | 4 ++ 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/managed/node-agent/resources/node-agent-installer.sh b/managed/node-agent/resources/node-agent-installer.sh index c82afb70baef..0f6d5807b109 100755 --- a/managed/node-agent/resources/node-agent-installer.sh +++ b/managed/node-agent/resources/node-agent-installer.sh @@ -17,6 +17,7 @@ SKIP_VERIFY_CERT="" #Disable node to Yugabyte Anywhere connection. DISABLE_EGRESS="false" SILENT_INSTALL="false" +AIRGAP_INSTALL="false" CERT_DIR="" CUSTOMER_ID="" NODE_NAME="" @@ -225,30 +226,40 @@ modify_firewall() { modify_selinux() { set +e if ! command -v semanage >/dev/null 2>&1; then - if command -v yum >/dev/null 2>&1; then - sudo yum install -y policycoreutils-python-utils - elif command -v apt-get >/dev/null 2>&1; then - sudo apt-get update -y - sudo apt-get install -y semanage-utils + if [ "$AIRGAP_INSTALL" = "true" ]; then + # The changes made with chcon are temporary in the sense that the context of the file + # altered with chcon goes back to default when restorecon is run. + # It should not even try to reach out to the repo. + sudo chcon -R -t bin_t "$NODE_AGENT_HOME" + else + if command -v yum >/dev/null 2>&1; then + sudo yum install -y policycoreutils-python-utils + elif command -v apt-get >/dev/null 2>&1; then + sudo apt-get update -y + sudo apt-get install -y semanage-utils + fi fi fi - sudo semanage port -lC | grep -F "$NODE_PORT" >/dev/null 2>&1 - if [ "$?" -ne 0 ]; then - sudo semanage port -a -t http_port_t -p tcp "$NODE_PORT" - fi - sudo semanage fcontext -lC | grep -F "$NODE_AGENT_HOME(/.*)?" >/dev/null 2>&1 - if [ "$?" -ne 0 ]; then - sudo semanage fcontext -a -t bin_t "$NODE_AGENT_HOME(/.*)?" + # Check if semanage was installed in the previous steps. + if command -v semanage >/dev/null 2>&1; then + sudo semanage port -lC | grep -F "$NODE_PORT" >/dev/null 2>&1 + if [ "$?" -ne 0 ]; then + sudo semanage port -a -t http_port_t -p tcp "$NODE_PORT" + fi + sudo semanage fcontext -lC | grep -F "$NODE_AGENT_HOME(/.*)?" >/dev/null 2>&1 + if [ "$?" -ne 0 ]; then + sudo semanage fcontext -a -t bin_t "$NODE_AGENT_HOME(/.*)?" + fi + sudo restorecon -ir "$NODE_AGENT_HOME" fi set -e - sudo restorecon -ir "$NODE_AGENT_HOME" } install_systemd_service() { if [ "$SE_LINUX_STATUS" = "Enforcing" ]; then modify_selinux - modify_firewall fi + modify_firewall echo "* Installing Node Agent Systemd Service" sudo tee "$SYSTEMD_DIR/$SERVICE_NAME" <<-EOF [Unit] @@ -298,6 +309,8 @@ Options: Username of the installation. A sudo user can install service for a non-sudo user. --skip_verify_cert (OPTIONAL) Specify to skip Yugabyte Anywhere server cert verification during install. + --airgap (OPTIONAL) + Specify to skip installing semanage utility. -h, --help Show usage. EOT @@ -470,6 +483,9 @@ while [[ $# -gt 0 ]]; do --silent) SILENT_INSTALL="true" ;; + --airgap) + AIRGAP_INSTALL="true" + ;; --node_name) NODE_NAME="$2" shift diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java index f239da8beec7..4325af779b1d 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java @@ -1505,6 +1505,7 @@ public SubTaskGroup createInstallNodeAgentTasks(Collection nodes) { public SubTaskGroup createInstallNodeAgentTasks( Collection nodes, boolean reinstall) { + Map nodeUuidProviderMap = new HashMap<>(); SubTaskGroup subTaskGroup = createSubTaskGroup(InstallNodeAgent.class.getSimpleName()); int serverPort = confGetter.getGlobalConf(GlobalConfKeys.nodeAgentServerPort); Universe universe = getUniverse(); @@ -1513,6 +1514,15 @@ public SubTaskGroup createInstallNodeAgentTasks( .forEach( n -> { InstallNodeAgent.Params params = new InstallNodeAgent.Params(); + Provider provider = + nodeUuidProviderMap.computeIfAbsent( + n.placementUuid, + k -> { + Cluster cluster = universe.getCluster(n.placementUuid); + return Provider.getOrBadRequest( + UUID.fromString(cluster.userIntent.provider)); + }); + params.airgap = provider.getAirGapInstall(); params.nodeName = n.nodeName; params.customerUuid = customer.getUuid(); params.azUuid = n.azUuid; diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallNodeAgent.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallNodeAgent.java index 1de63d20e341..5c3454f16653 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallNodeAgent.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallNodeAgent.java @@ -57,6 +57,7 @@ public static class Params extends NodeTaskParams { public String nodeAgentHome; public UUID customerUuid; public boolean reinstall; + public boolean airgap; } @Override @@ -165,6 +166,9 @@ public void run() { sb.append(" --node_name ").append(node.getNodeName()); sb.append(" --node_ip ").append(node.cloudInfo.private_ip); sb.append(" --node_port ").append(String.valueOf(taskParams().nodeAgentPort)); + if (taskParams().airgap) { + sb.append(" --airgap"); + } // Give executable permission to node-agent path. sb.append(" && chmod 755 /root ").append(taskParams().nodeAgentHome); // Remove the unused installer script.