From 00530e10a8e3e1ac799a9518ca30aadb9fde239e Mon Sep 17 00:00:00 2001 From: "Arora, Sakshi" Date: Tue, 9 Jul 2024 13:18:01 +0530 Subject: [PATCH] roce plugin changes --- input/config/ubuntu/20.04/roce_plugin.json | 6 ++- input/config/ubuntu/22.04/roce_plugin.json | 6 ++- input/roce_plugin_config.yml | 54 +++++++++++++++---- .../tasks/apply_network_services.yml | 46 ++++++++++++++-- .../tasks/validate_roce_plugin_config.yml | 14 ++--- .../interface_attachment_definition.yml.j2 | 24 +++++++-- scheduler/roles/k8s_roce_deploy/vars/main.yml | 7 ++- 7 files changed, 118 insertions(+), 39 deletions(-) diff --git a/input/config/ubuntu/20.04/roce_plugin.json b/input/config/ubuntu/20.04/roce_plugin.json index 6057d5c22..916b0398c 100644 --- a/input/config/ubuntu/20.04/roce_plugin.json +++ b/input/config/ubuntu/20.04/roce_plugin.json @@ -10,13 +10,15 @@ "package": "whereabouts", "url": "https://github.com/k8snetworkplumbingwg/whereabouts.git", "type": "git", - "version": "v0.6.3" + "version": "master", + "commit": "638d58" }, { "package": "k8s-rdma-shared-dev-plugin", "url": "https://github.com/Mellanox/k8s-rdma-shared-dev-plugin.git", "type": "git", - "version": "master" + "version": "master", + "commit": "c94b2cef" }, { "package": "ghcr.io/k8snetworkplumbingwg/multus-cni", diff --git a/input/config/ubuntu/22.04/roce_plugin.json b/input/config/ubuntu/22.04/roce_plugin.json index 6057d5c22..916b0398c 100644 --- a/input/config/ubuntu/22.04/roce_plugin.json +++ b/input/config/ubuntu/22.04/roce_plugin.json @@ -10,13 +10,15 @@ "package": "whereabouts", "url": "https://github.com/k8snetworkplumbingwg/whereabouts.git", "type": "git", - "version": "v0.6.3" + "version": "master", + "commit": "638d58" }, { "package": "k8s-rdma-shared-dev-plugin", "url": "https://github.com/Mellanox/k8s-rdma-shared-dev-plugin.git", "type": "git", - "version": "master" + "version": "master", + "commit": "c94b2cef" }, { "package": "ghcr.io/k8snetworkplumbingwg/multus-cni", diff --git a/input/roce_plugin_config.yml b/input/roce_plugin_config.yml index baf4b6ba4..cab5f5562 100644 --- a/input/roce_plugin_config.yml +++ b/input/roce_plugin_config.yml @@ -14,36 +14,68 @@ --- # This config file will be used for deploying k8s roce plugin. +# Roce plugin is supported only on Ubuntu OS. -# Define network interfaces with their respective IP ranges and gateways which should be assigned inside the pod in this dictionary +# Define network interfaces with their respective IP ranges which should be assigned inside the pod in this dictionary # Number of entries in this dictionary should be equal to number of bcm roce interfaces which must be used for roce pod. # VLAN nics are not supported for roce plugin deployment -# name: The name of the interface of the roce nic -# range: The IP range for this interface which should be assigned inside rocepod -# gateway: The gateway for this interface which should be assigned inside rocepod +# A limit of 8 roce interfaces is supported by the rocepod deployed. +# name (Mandatory): The name of the interface of the roce nic +# range (Mandatory): The IP range for this interface which should be assigned inside rocepod. It is specified in CIDR notation. +# range_start (Optional): This specifies the starting IP address within the defined range for assigning IPs to network interfaces. +# range_end (Optional): This specifies the ending IP address within the defined range for assigning IPs to network interfaces. +# gateway (Optional): This specifies the IP address of the gateway for the network. +# route (Optional): This specifies additional routing rules for the network interface. Routes determine the paths that packets take to reach specific networks or hosts. +# Omnia does not validate the inputs, it is user's responsibility to provide inputs for the required parameters. +# Eg: If user wants to restrict the IP range, range_start and range_end should be provided with range. +# Eg2: If gateway is to be set, then user should provide gateway and route along with range. interfaces: - name: eth1 range: 192.168.1.0/24 + range_start: + range_end: gateway: 192.168.1.1 + route: 192.168.1.0/24 - name: eth2 range: 192.168.2.0/24 - gateway: 192.168.2.1 + range_start: + range_end: + gateway: + route: - name: eth3 range: 192.168.3.0/24 - gateway: 192.168.3.1 + range_start: + range_end: + gateway: + route: - name: eth4 range: 192.168.4.0/24 - gateway: 192.168.4.1 + range_start: + range_end: + gateway: + route: - name: eth5 range: 192.168.5.0/24 - gateway: 192.168.5.1 + range_start: + range_end: + gateway: + route: - name: eth6 range: 192.168.6.0/24 - gateway: 192.168.6.1 + range_start: + range_end: + gateway: + route: - name: eth7 range: 192.168.7.0/24 - gateway: 192.168.7.1 + range_start: + range_end: + gateway: + route: - name: eth8 range: 192.168.8.0/24 - gateway: 192.168.8.1 \ No newline at end of file + range_start: + range_end: + gateway: + route: \ No newline at end of file diff --git a/scheduler/roles/k8s_roce_deploy/tasks/apply_network_services.yml b/scheduler/roles/k8s_roce_deploy/tasks/apply_network_services.yml index f6f8e1733..9836d62f2 100644 --- a/scheduler/roles/k8s_roce_deploy/tasks/apply_network_services.yml +++ b/scheduler/roles/k8s_roce_deploy/tasks/apply_network_services.yml @@ -13,23 +13,47 @@ # limitations under the License. --- +- name: Set variables + ansible.builtin.set_fact: + whereabouts_plugin_commit: false + rdma_plugin_commit: false + +# -------------------------MULTUS PLUGIN-------------------------- - name: Apply multus pod network ansible.builtin.command: "kubectl apply -f {{ multus_pod_network_url }}" changed_when: true +# -------------------------WHEREABOUTS PLUGIN---------------------------- - name: Get whereabouts plugin git folder ansible.builtin.get_url: url: "{{ whereabouts_git_url }}" dest: "{{ whereabouts_dest_folder }}" - mode: "{{ file_permission }}" + mode: "{{ git_permissions }}" - name: Unarchive whereabouts git folder ansible.builtin.unarchive: src: "{{ whereabouts_dest_folder }}" dest: "{{ tmp_folder }}" - mode: "{{ dir_permission }}" + mode: "{{ git_permissions }}" remote_src: true +- name: Check if commit id is mentioned for whereabouts plugin + ansible.builtin.set_fact: + whereabouts_plugin_commit: true + whereabouts_commit_id: "{{ item.commit }}" + loop: "{{ hostvars['localhost']['roce_plugin_packages_json']['roce_plugin']['cluster'] }}" + when: + - item.type == 'git' + - "'whereabouts' in item.package" + - "item.commit is defined" + +- name: Checkout specific commit for whereabouts plugin + ansible.builtin.command: "git checkout {{ whereabouts_commit_id }}" # noqa: command-instead-of-module + when: whereabouts_plugin_commit + changed_when: false + args: + chdir: "{{ whereabouts_folder }}" + - name: Apply whereabouts crds ansible.builtin.command: "kubectl apply -f {{ wherabout_crds_folder }}" changed_when: true @@ -40,10 +64,13 @@ -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/imagePullPolicy", "value":"IfNotPresent"}]' changed_when: false +# ------------------NETWORK ATTACHMENT DEFINITIONS------------------------ + - name: Create macvlan configs with NetworkAttachmentDefinitions ansible.builtin.command: "kubectl apply -f {{ roce_nad_tmp_folder }}" changed_when: true +# ------------------------RDMA PLUGIN----------------------------- - name: Get rdma-plugin git folder ansible.builtin.get_url: url: "{{ rdma_plugin_git_url }}" @@ -57,9 +84,20 @@ mode: "{{ git_permissions }}" remote_src: true -- name: Checkout specific commit for rdma_plugin - ansible.builtin.command: "git checkout {{ rdma_plugin_commit_id }}" # noqa: command-instead-of-module +- name: Check if commit id is mentioned for rdma plugin + ansible.builtin.set_fact: + rdma_plugin_commit: true + rdma_commit_id: "{{ item.commit }}" + loop: "{{ hostvars['localhost']['roce_plugin_packages_json']['roce_plugin']['cluster'] }}" + when: + - item.type == 'git' + - "'rdma-shared-dev-plugin' in item.package" + - "item.commit is defined" + +- name: Checkout specific commit for rdma plugin + ansible.builtin.command: "git checkout {{ rdma_commit_id }}" # noqa: command-instead-of-module changed_when: false + when: rdma_plugin_commit args: chdir: "{{ rdma_plugin_folder }}" diff --git a/scheduler/roles/k8s_roce_deploy/tasks/validate_roce_plugin_config.yml b/scheduler/roles/k8s_roce_deploy/tasks/validate_roce_plugin_config.yml index 074ed0be8..2e7d3717e 100644 --- a/scheduler/roles/k8s_roce_deploy/tasks/validate_roce_plugin_config.yml +++ b/scheduler/roles/k8s_roce_deploy/tasks/validate_roce_plugin_config.yml @@ -30,13 +30,12 @@ fail_msg: "{{ name_unique_fail_msg }}" success_msg: "{{ name_unique_success_msg }}" -- name: Ensure all interfaces have range and gateway defined +- name: Ensure all interfaces have range defined ansible.builtin.assert: that: - interfaces | map(attribute='range') | select('defined') | list | length == interfaces | length - - interfaces | map(attribute='gateway') | select('defined') | list | length == interfaces | length - fail_msg: "{{ range_gateway_definition_fail_msg }}" - success_msg: "{{ range_gateway_definition_success_msg }}" + fail_msg: "{{ range_definition_fail_msg }}" + success_msg: "{{ range_definition_success_msg }}" - name: Check for unique range values ansible.builtin.assert: @@ -44,10 +43,3 @@ - interfaces | map(attribute='range') | length == interfaces | map(attribute='range') | unique | length fail_msg: "{{ range_unique_fail_msg }}" success_msg: "{{ range_unique_success_msg }}" - -- name: Check for unique gateway values - ansible.builtin.assert: - that: - - interfaces | map(attribute='gateway') | length == interfaces | map(attribute='gateway') | unique | length - fail_msg: "{{ gateway_unique_fail_msg }}" - success_msg: "{{ gateway_unique_success_msg }}" diff --git a/scheduler/roles/k8s_roce_deploy/templates/interface_attachment_definition.yml.j2 b/scheduler/roles/k8s_roce_deploy/templates/interface_attachment_definition.yml.j2 index e857c0f09..c49f52bb4 100644 --- a/scheduler/roles/k8s_roce_deploy/templates/interface_attachment_definition.yml.j2 +++ b/scheduler/roles/k8s_roce_deploy/templates/interface_attachment_definition.yml.j2 @@ -1,3 +1,20 @@ +{% set ipam_fields = [] %} +{% if item.1.range %} + {% set _ = ipam_fields.append('"range": "' ~ item.1.range ~ '"') %} +{% endif %} +{% if item.1.range_start %} + {% set _ = ipam_fields.append('"range_start": "' ~ item.1.range_start ~ '"') %} +{% endif %} +{% if item.1.range_end %} + {% set _ = ipam_fields.append('"range_end": "' ~ item.1.range_end ~ '"') %} +{% endif %} +{% if item.1.route %} + {% set _ = ipam_fields.append('"routes": [{ "dst": "' ~ item.1.route ~ '" }]') %} +{% endif %} +{% if item.1.gateway %} + {% set _ = ipam_fields.append('"gateway": "' ~ item.1.gateway ~ '"') %} +{% endif %} + apiVersion: "k8s.cni.cncf.io/v1" kind: NetworkAttachmentDefinition metadata: @@ -11,10 +28,7 @@ spec: "mode": "bridge", "ipam": { "type": "whereabouts", - "range": "{{ item.1.range }}", - "routes": [ - { "dst": "0.0.0.0/0" } - ], - "gateway": "{{ item.1.gateway }}" + {% if ipam_fields | length > 0 %}{{ ipam_fields | join(',\n ') }}{% endif %} + } }' diff --git a/scheduler/roles/k8s_roce_deploy/vars/main.yml b/scheduler/roles/k8s_roce_deploy/vars/main.yml index e39144ad0..f97b8871b 100644 --- a/scheduler/roles/k8s_roce_deploy/vars/main.yml +++ b/scheduler/roles/k8s_roce_deploy/vars/main.yml @@ -39,12 +39,10 @@ name_definition_fail_msg: "One or more interfaces are missing 'name'" name_definition_success_msg: "All interfaces have 'name' defined" name_unique_fail_msg: "Interface names are not unique" name_unique_success_msg: "All interface names are unique" -range_gateway_definition_fail_msg: "One or more interfaces are missing 'range' or 'gateway'" -range_gateway_definition_success_msg: "All interfaces have 'range' and 'gateway' defined" +range_definition_fail_msg: "One or more interfaces are missing value for variable 'range' in roce_plugin_config.yml " +range_definition_success_msg: "All interfaces have 'range'" range_unique_fail_msg: "Range values are not unique" range_unique_success_msg: "All range values are unique" -gateway_unique_fail_msg: "Gateway values are not unique" -gateway_unique_success_msg: "All gateway values are unique" # Usage: create_config_files.yml rdma_patch_file_dest: "/tmp/rdma-plugin.patch" @@ -56,6 +54,7 @@ dir_permission: "0755" multus_pod_network_url: "{{ hostvars['localhost']['offline_manifest_path'] }}/multus-daemonset.yaml" whereabouts_git_url: "{{ hostvars['localhost']['offline_git_path'] }}/whereabouts.tar.gz" whereabouts_dest_folder: "/tmp/whereabouts.tar.gz" +whereabouts_folder: "/tmp/whereabouts" tmp_folder: "/tmp" wherabout_crds_folder: "/tmp/whereabouts/doc/crds/" rdma_plugin_git_url: "{{ hostvars['localhost']['offline_git_path'] }}/k8s-rdma-shared-dev-plugin.tar.gz"