[IPv6 only] Fix ipv6_only fixture to remove ipv4 from snmp agent addr…

…ess (sonic-net#12325)

Description of PR
Fix the ipv6 fixture to handle SNMP_AGENT_ADDRESS_CONFIG in config.
Revise the assert_addr_in_ifconfig to assert_addr_in_output to make it more reusable.
Summary:
Fix feature interaction issue between [sonic-net/SONiC#12010:https://github.com/sonic-net/SONiC/pull/12010] and [sonic-net/SONiC#17045:https://github.com/sonic-net/SONiC/pull/17045]

Approach
What is the motivation for this PR?
current convert_and_restore_config_db_to_ipv6_only() fixture - removes the IPv4 mgmt address, but the IPV4 mgmt address continues to be in SNMP_AGENT_ADDRESS_CONFIG table. Upon config reload when snmpd comes up, it sees ipv4 and ipv6 mgmt addresses in SNMP_AGENT_ADDRESS_CONFIG table and snmpd fails to come up as mgmt interface does not have ipv4 address assigned to the interface.

How did you do it?
Fixture will check all IPs in the SNMP_AGENT_ADDRESS_CONFIG table and remove the IPv4 addresses.

How did you verify/test it?
Run on physical testbeds, with dual-stack mgmt-ip addresses testbed and only with IPv6/IPv4 mgmt-ip address testbed.

ip/test_mgmt_ipv6_only.py::test_bgp_facts_ipv6_only[-] PASSED [ 20%]
ip/test_mgmt_ipv6_only.py::test_show_features_ipv6_only[-] PASSED [ 40%]
ip/test_mgmt_ipv6_only.py::test_image_download_ipv6_only[-] SKIPPED (Cannot get image url) [ 60%]
ip/test_mgmt_ipv6_only.py::test_syslog_ipv6_only[-] PASSED [ 80%]
ip/test_mgmt_ipv6_only.py::test_syslog_ipv6_only[-] PASSED [100%]

co-authorized by: jianquanye@microsoft.com

tests/common/fixtures/duthost_utils.py

@@ -637,6 +637,14 @@ def convert_and_restore_config_db_to_ipv6_only(duthosts):
  # ]
  # }
  # }
+ #
+ # Sample SNMP_AGENT_ADDRESS_CONFIG:
+ # "SNMP_AGENT_ADDRESS_CONFIG": {
+ # "10.1.0.32|161|": {},
+ # "10.250.0.101|161|": {},
+ # "FC00:1::32|161|": {},
+ # "fec0::ffff:afa:1|161|": {}
+ # }, },
 
  # duthost_name: config_db_modified
  config_db_modified: Dict[str, bool] = {duthost.hostname: False
@@ -700,22 +708,66 @@ def convert_and_restore_config_db_to_ipv6_only(duthosts):
  duthost.shell(f"""jq 'del(."MGMT_INTERFACE"."{key}")' {config_db_file} > temp.json"""
  f"""&& mv temp.json {config_db_file}""", module_ignore_errors=True)
  config_db_modified[duthost.hostname] = True
- config_reload(duthost, wait=120)
+
+ # Save both IPv4 and IPv6 SNMP address for verification purpose.
+ snmp_ipv4_address: Dict[str, List] = {duthost.hostname: []
+ for duthost in duthosts.nodes}
+ snmp_ipv6_address: Dict[str, List] = {duthost.hostname: []
+ for duthost in duthosts.nodes}
+ for duthost in duthosts.nodes:
+ snmp_address = json.loads(duthost.shell(f"jq '.SNMP_AGENT_ADDRESS_CONFIG' {config_db_file}",
+ module_ignore_errors=True)["stdout"])
+ # In case device doesn't have SNMP_AGENT_CONFIG: this could happen if
+ # DUT is running old image.
+ if not snmp_address:
+ logger.info(f"No SNMP_AGENT_ADDRESS_CONFIG found in host[{duthost.hostname}] {config_db_file}, continue.")
+ continue
+ for key in list(snmp_address):
+ ip_addr = key.split("|")[0]
+ if ip_addr:
+ if valid_ipv4(ip_addr):
+ snmp_ipv4_address[duthost.hostname].append(ip_addr)
+ logger.info(f"Removing host[{duthost.hostname}] SNMP IPv4 address {ip_addr}")
+ duthost.shell(f"""jq 'del(."SNMP_AGENT_ADDRESS_CONFIG"."{key}")' {config_db_file} > temp.json"""
+ f"""&& mv temp.json {config_db_file}""", module_ignore_errors=True)
+ config_db_modified[duthost.hostname] = True
+ elif valid_ipv6(ip_addr):
+ snmp_ipv6_address[duthost.hostname].append(ip_addr.lower())
+
+ # Do config_reload after processing BOTH SNMP and MGMT config
+ for duthost in duthosts.nodes:
+ if config_db_modified[duthost.hostname]:
+ logger.info(f"config changed. Doing config reload for {duthost.hostname}")
+ config_reload(duthost, wait=120)
  duthosts.reset()
 
  # Verify mgmt-interface status
  mgmt_intf_name = "eth0"
  for duthost in duthosts.nodes:
  logger.info(f"Checking host[{duthost.hostname}] mgmt interface[{mgmt_intf_name}]")
  mgmt_intf_ifconfig = duthost.shell(f"ifconfig {mgmt_intf_name}", module_ignore_errors=True)["stdout"]
- assert_addr_in_ifconfig(addr_set=ipv4_address, hostname=duthost.hostname,
- expect_exists=False, ifconfig_output=mgmt_intf_ifconfig)
- assert_addr_in_ifconfig(addr_set=ipv6_address, hostname=duthost.hostname,
- expect_exists=True, ifconfig_output=mgmt_intf_ifconfig)
+ assert_addr_in_output(addr_set=ipv4_address, hostname=duthost.hostname,
+ expect_exists=False, cmd_output=mgmt_intf_ifconfig,
+ cmd_desc="ifconfig")
+ assert_addr_in_output(addr_set=ipv6_address, hostname=duthost.hostname,
+ expect_exists=True, cmd_output=mgmt_intf_ifconfig,
+ cmd_desc="ifconfig")
+
+ # Verify SNMP address status
+ for duthost in duthosts.nodes:
+ logger.info(f"Checking host[{duthost.hostname}] SNMP status in netstat output")
+ snmp_netstat_output = duthost.shell("sudo netstat -tulnpW | grep snmpd",
+ module_ignore_errors=True)["stdout"]
+ assert_addr_in_output(addr_set=snmp_ipv4_address, hostname=duthost.hostname,
+ expect_exists=False, cmd_output=snmp_netstat_output,
+ cmd_desc="netstat")
+ assert_addr_in_output(addr_set=snmp_ipv6_address, hostname=duthost.hostname,
+ expect_exists=True, cmd_output=snmp_netstat_output,
+ cmd_desc="netstat")
 
  yield
 
- # Recover IPv4 mgmt-ip
+ # Recover IPv4 mgmt-ip and other config (SNMP_ADDRESS, etc.)
  for duthost in duthosts.nodes:
  if config_db_modified[duthost.hostname]:
  logger.info(f"Restore {config_db_file} with {config_db_bak_file} on {duthost.hostname}")
@@ -727,30 +779,46 @@ def convert_and_restore_config_db_to_ipv6_only(duthosts):
  for duthost in duthosts.nodes:
  logger.info(f"Checking host[{duthost.hostname}] mgmt interface[{mgmt_intf_name}]")
  mgmt_intf_ifconfig = duthost.shell(f"ifconfig {mgmt_intf_name}", module_ignore_errors=True)["stdout"]
- assert_addr_in_ifconfig(addr_set=ipv4_address, hostname=duthost.hostname,
- expect_exists=True, ifconfig_output=mgmt_intf_ifconfig)
- assert_addr_in_ifconfig(addr_set=ipv6_address, hostname=duthost.hostname,
- expect_exists=True, ifconfig_output=mgmt_intf_ifconfig)
+ assert_addr_in_output(addr_set=ipv4_address, hostname=duthost.hostname,
+ expect_exists=True, cmd_output=mgmt_intf_ifconfig,
+ cmd_desc="ifconfig")
+ assert_addr_in_output(addr_set=ipv6_address, hostname=duthost.hostname,
+ expect_exists=True, cmd_output=mgmt_intf_ifconfig,
+ cmd_desc="ifconfig")
+
+ # Verify SNMP address status
+ for duthost in duthosts.nodes:
+ logger.info(f"Checking host[{duthost.hostname}] SNMP status in netstat output")
+ snmp_netstat_output = duthost.shell("sudo netstat -tulnpW | grep snmpd",
+ module_ignore_errors=True)["stdout"]
+ assert_addr_in_output(addr_set=snmp_ipv4_address, hostname=duthost.hostname,
+ expect_exists=True, cmd_output=snmp_netstat_output,
+ cmd_desc="netstat")
+ assert_addr_in_output(addr_set=snmp_ipv6_address, hostname=duthost.hostname,
+ expect_exists=True, cmd_output=snmp_netstat_output,
+ cmd_desc="netstat")
 
 
-def assert_addr_in_ifconfig(addr_set: Dict[str, List], hostname: str, expect_exists: bool, ifconfig_output: str):
+def assert_addr_in_output(addr_set: Dict[str, List], hostname: str,
+ expect_exists: bool, cmd_output: str, cmd_desc: str):
  """
- Assert the address status in the ifconfig output,
+ Assert the address status in the command output,
  if status not as expected, assert as failure
 
  @param addr_set: addr_set, key is dut hostname, value is the list of ip addresses
  @param hostname: hostname
  @param expect_exists: Expectation of the ip,
- True means expect all ip addresses in addr_set appears in the output of ifconfig
- False means expect no ip addresses in addr_set appears in the output of ifconfig
- @param ifconfig_output: output of 'ifconfig'
+ True means expect all ip addresses in addr_set appears in the output of cmd_output
+ False means expect no ip addresses in addr_set appears in the output of cmd_output
+ @param cmd_output: command output
+ @param cmd_desc: command description, used for logging purpose.
  """
  for addr in addr_set[hostname]:
  if expect_exists:
- pytest_assert(addr in ifconfig_output,
- f"{addr} not appeared in {hostname} mgmt interface")
- logger.info(f"{addr} exists in the output of ifconfig")
+ pytest_assert(addr in cmd_output,
+ f"{addr} not appeared in {hostname} {cmd_desc}")
+ logger.info(f"{addr} exists in the output of {cmd_desc}")
  else:
- pytest_assert(addr not in ifconfig_output,
- f"{hostname} mgmt interface still with addr {addr}")
- logger.info(f"{addr} not exists in the output of ifconfig which is expected")
+ pytest_assert(addr not in cmd_output,
+ f"{hostname} {cmd_desc} still with addr {addr}")
+ logger.info(f"{addr} not exists in the output of {cmd_desc} which is expected")