Merge pull request #2292 from sakshiarora13/release_1.6

roce plugin changes

input/config/ubuntu/20.04/roce_plugin.json

@@ -10,13 +10,15 @@
  "package": "whereabouts",
  "url": "https://github.com/k8snetworkplumbingwg/whereabouts.git",
  "type": "git",
- "version": "v0.6.3"
+ "version": "master",
+ "commit": "638d58"
  },
  {
  "package": "k8s-rdma-shared-dev-plugin",
  "url": "https://github.com/Mellanox/k8s-rdma-shared-dev-plugin.git",
  "type": "git",
- "version": "master"
+ "version": "master",
+ "commit": "c94b2cef"
  },
  {
  "package": "ghcr.io/k8snetworkplumbingwg/multus-cni",

input/config/ubuntu/22.04/roce_plugin.json

@@ -10,13 +10,15 @@
  "package": "whereabouts",
  "url": "https://github.com/k8snetworkplumbingwg/whereabouts.git",
  "type": "git",
- "version": "v0.6.3"
+ "version": "master",
+ "commit": "638d58"
  },
  {
  "package": "k8s-rdma-shared-dev-plugin",
  "url": "https://github.com/Mellanox/k8s-rdma-shared-dev-plugin.git",
  "type": "git",
- "version": "master"
+ "version": "master",
+ "commit": "c94b2cef"
  },
  {
  "package": "ghcr.io/k8snetworkplumbingwg/multus-cni",

input/roce_plugin_config.yml

@@ -14,36 +14,68 @@
 ---
 
 # This config file will be used for deploying k8s roce plugin.
+# Roce plugin is supported only on Ubuntu OS.
 
 
-# Define network interfaces with their respective IP ranges and gateways which should be assigned inside the pod in this dictionary
+# Define network interfaces with their respective IP ranges which should be assigned inside the pod in this dictionary
 # Number of entries in this dictionary should be equal to number of bcm roce interfaces which must be used for roce pod.
 # VLAN nics are not supported for roce plugin deployment
-# name: The name of the interface of the roce nic
-# range: The IP range for this interface which should be assigned inside rocepod
-# gateway: The gateway for this interface which should be assigned inside rocepod
+# A limit of 8 roce interfaces is supported by the rocepod deployed.
+# name (Mandatory): The name of the interface of the roce nic
+# range (Mandatory): The IP range for this interface which should be assigned inside rocepod. It is specified in CIDR notation.
+# range_start (Optional): This specifies the starting IP address within the defined range for assigning IPs to network interfaces.
+# range_end (Optional): This specifies the ending IP address within the defined range for assigning IPs to network interfaces.
+# gateway (Optional): This specifies the IP address of the gateway for the network.
+# route (Optional): This specifies additional routing rules for the network interface. Routes determine the paths that packets take to reach specific networks or hosts.
+# Omnia does not validate the inputs, it is user's responsibility to provide inputs for the required parameters.
+# Eg: If user wants to restrict the IP range, range_start and range_end should be provided with range.
+# Eg2: If gateway is to be set, then user should provide gateway and route along with range.
 interfaces:
  - name: eth1
  range: 192.168.1.0/24
+ range_start:
+ range_end:
  gateway: 192.168.1.1
+ route: 192.168.1.0/24
  - name: eth2
  range: 192.168.2.0/24
- gateway: 192.168.2.1
+ range_start:
+ range_end:
+ gateway: 
+ route:
  - name: eth3
  range: 192.168.3.0/24
- gateway: 192.168.3.1
+ range_start:
+ range_end:
+ gateway: 
+ route:
  - name: eth4
  range: 192.168.4.0/24
- gateway: 192.168.4.1
+ range_start:
+ range_end:
+ gateway: 
+ route:
  - name: eth5
  range: 192.168.5.0/24
- gateway: 192.168.5.1
+ range_start:
+ range_end:
+ gateway: 
+ route:
  - name: eth6
  range: 192.168.6.0/24
- gateway: 192.168.6.1
+ range_start:
+ range_end:
+ gateway: 
+ route:
  - name: eth7
  range: 192.168.7.0/24
- gateway: 192.168.7.1
+ range_start:
+ range_end:
+ gateway: 
+ route:
  - name: eth8
  range: 192.168.8.0/24
- gateway: 192.168.8.1
+ range_start:
+ range_end:
+ gateway: 
+ route:

scheduler/roles/k8s_roce_deploy/tasks/apply_network_services.yml

@@ -13,23 +13,47 @@
 # limitations under the License.
 ---
 
+- name: Set variables
+ ansible.builtin.set_fact:
+ whereabouts_plugin_commit: false
+ rdma_plugin_commit: false
+
+# -------------------------MULTUS PLUGIN--------------------------
 - name: Apply multus pod network
  ansible.builtin.command: "kubectl apply -f {{ multus_pod_network_url }}"
  changed_when: true
 
+# -------------------------WHEREABOUTS PLUGIN----------------------------
 - name: Get whereabouts plugin git folder
  ansible.builtin.get_url:
  url: "{{ whereabouts_git_url }}"
  dest: "{{ whereabouts_dest_folder }}"
- mode: "{{ file_permission }}"
+ mode: "{{ git_permissions }}"
 
 - name: Unarchive whereabouts git folder
  ansible.builtin.unarchive:
  src: "{{ whereabouts_dest_folder }}"
  dest: "{{ tmp_folder }}"
- mode: "{{ dir_permission }}"
+ mode: "{{ git_permissions }}"
  remote_src: true
 
+- name: Check if commit id is mentioned for whereabouts plugin
+ ansible.builtin.set_fact:
+ whereabouts_plugin_commit: true
+ whereabouts_commit_id: "{{ item.commit }}"
+ loop: "{{ hostvars['localhost']['roce_plugin_packages_json']['roce_plugin']['cluster'] }}"
+ when:
+ - item.type == 'git'
+ - "'whereabouts' in item.package"
+ - "item.commit is defined"
+
+- name: Checkout specific commit for whereabouts plugin
+ ansible.builtin.command: "git checkout {{ whereabouts_commit_id }}" # noqa: command-instead-of-module
+ when: whereabouts_plugin_commit
+ changed_when: false
+ args:
+ chdir: "{{ whereabouts_folder }}"
+
 - name: Apply whereabouts crds
  ansible.builtin.command: "kubectl apply -f {{ wherabout_crds_folder }}"
  changed_when: true
@@ -40,10 +64,13 @@
  -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/imagePullPolicy", "value":"IfNotPresent"}]'
  changed_when: false
 
+# ------------------NETWORK ATTACHMENT DEFINITIONS------------------------
+
 - name: Create macvlan configs with NetworkAttachmentDefinitions
  ansible.builtin.command: "kubectl apply -f {{ roce_nad_tmp_folder }}"
  changed_when: true
 
+# ------------------------RDMA PLUGIN-----------------------------
 - name: Get rdma-plugin git folder
  ansible.builtin.get_url:
  url: "{{ rdma_plugin_git_url }}"
@@ -57,9 +84,20 @@
  mode: "{{ git_permissions }}"
  remote_src: true
 
-- name: Checkout specific commit for rdma_plugin
- ansible.builtin.command: "git checkout {{ rdma_plugin_commit_id }}" # noqa: command-instead-of-module
+- name: Check if commit id is mentioned for rdma plugin
+ ansible.builtin.set_fact:
+ rdma_plugin_commit: true
+ rdma_commit_id: "{{ item.commit }}"
+ loop: "{{ hostvars['localhost']['roce_plugin_packages_json']['roce_plugin']['cluster'] }}"
+ when:
+ - item.type == 'git'
+ - "'rdma-shared-dev-plugin' in item.package"
+ - "item.commit is defined"
+
+- name: Checkout specific commit for rdma plugin
+ ansible.builtin.command: "git checkout {{ rdma_commit_id }}" # noqa: command-instead-of-module
  changed_when: false
+ when: rdma_plugin_commit
  args:
  chdir: "{{ rdma_plugin_folder }}"
 

scheduler/roles/k8s_roce_deploy/tasks/validate_roce_plugin_config.yml

@@ -30,24 +30,16 @@
  fail_msg: "{{ name_unique_fail_msg }}"
  success_msg: "{{ name_unique_success_msg }}"
 
-- name: Ensure all interfaces have range and gateway defined
+- name: Ensure all interfaces have range defined
  ansible.builtin.assert:
  that:
  - interfaces | map(attribute='range') | select('defined') | list | length == interfaces | length
- - interfaces | map(attribute='gateway') | select('defined') | list | length == interfaces | length
- fail_msg: "{{ range_gateway_definition_fail_msg }}"
- success_msg: "{{ range_gateway_definition_success_msg }}"
+ fail_msg: "{{ range_definition_fail_msg }}"
+ success_msg: "{{ range_definition_success_msg }}"
 
 - name: Check for unique range values
  ansible.builtin.assert:
  that:
  - interfaces | map(attribute='range') | length == interfaces | map(attribute='range') | unique | length
  fail_msg: "{{ range_unique_fail_msg }}"
  success_msg: "{{ range_unique_success_msg }}"
-
-- name: Check for unique gateway values
- ansible.builtin.assert:
- that:
- - interfaces | map(attribute='gateway') | length == interfaces | map(attribute='gateway') | unique | length
- fail_msg: "{{ gateway_unique_fail_msg }}"
- success_msg: "{{ gateway_unique_success_msg }}"

scheduler/roles/k8s_roce_deploy/templates/interface_attachment_definition.yml.j2

@@ -1,3 +1,20 @@
+{% set ipam_fields = [] %}
+{% if item.1.range %}
+ {% set _ = ipam_fields.append('"range": "' ~ item.1.range ~ '"') %}
+{% endif %}
+{% if item.1.range_start %}
+ {% set _ = ipam_fields.append('"range_start": "' ~ item.1.range_start ~ '"') %}
+{% endif %}
+{% if item.1.range_end %}
+ {% set _ = ipam_fields.append('"range_end": "' ~ item.1.range_end ~ '"') %}
+{% endif %}
+{% if item.1.route %}
+ {% set _ = ipam_fields.append('"routes": [{ "dst": "' ~ item.1.route ~ '" }]') %}
+{% endif %}
+{% if item.1.gateway %}
+ {% set _ = ipam_fields.append('"gateway": "' ~ item.1.gateway ~ '"') %}
+{% endif %}
+
 apiVersion: "k8s.cni.cncf.io/v1"
 kind: NetworkAttachmentDefinition
 metadata:
@@ -11,10 +28,7 @@ spec:
  "mode": "bridge",
  "ipam": {
  "type": "whereabouts",
- "range": "{{ item.1.range }}",
- "routes": [
- { "dst": "0.0.0.0/0" }
- ],
- "gateway": "{{ item.1.gateway }}"
+ {% if ipam_fields | length > 0 %}{{ ipam_fields | join(',\n ') }}{% endif %}
+
  }
  }'

scheduler/roles/k8s_roce_deploy/vars/main.yml

@@ -39,12 +39,10 @@ name_definition_fail_msg: "One or more interfaces are missing 'name'"
 name_definition_success_msg: "All interfaces have 'name' defined"
 name_unique_fail_msg: "Interface names are not unique"
 name_unique_success_msg: "All interface names are unique"
-range_gateway_definition_fail_msg: "One or more interfaces are missing 'range' or 'gateway'"
-range_gateway_definition_success_msg: "All interfaces have 'range' and 'gateway' defined"
+range_definition_fail_msg: "One or more interfaces are missing value for variable 'range' in roce_plugin_config.yml "
+range_definition_success_msg: "All interfaces have 'range'"
 range_unique_fail_msg: "Range values are not unique"
 range_unique_success_msg: "All range values are unique"
-gateway_unique_fail_msg: "Gateway values are not unique"
-gateway_unique_success_msg: "All gateway values are unique"
 
 # Usage: create_config_files.yml
 rdma_patch_file_dest: "/tmp/rdma-plugin.patch"
@@ -56,6 +54,7 @@ dir_permission: "0755"
 multus_pod_network_url: "{{ hostvars['localhost']['offline_manifest_path'] }}/multus-daemonset.yaml"
 whereabouts_git_url: "{{ hostvars['localhost']['offline_git_path'] }}/whereabouts.tar.gz"
 whereabouts_dest_folder: "/tmp/whereabouts.tar.gz"
+whereabouts_folder: "/tmp/whereabouts"
 tmp_folder: "/tmp"
 wherabout_crds_folder: "/tmp/whereabouts/doc/crds/"
 rdma_plugin_git_url: "{{ hostvars['localhost']['offline_git_path'] }}/k8s-rdma-shared-dev-plugin.tar.gz"