diff --git a/tests/voq/test_fabric_cli_and_db.py b/tests/voq/test_fabric_cli_and_db.py index de3e39891dd..e72d9de8a75 100644 --- a/tests/voq/test_fabric_cli_and_db.py +++ b/tests/voq/test_fabric_cli_and_db.py @@ -2,6 +2,7 @@ from tests.common.helpers.assertions import pytest_assert import logging import random + logger = logging.getLogger(__name__) # This test only runs on t2 systems. pytestmark = [ @@ -16,6 +17,7 @@ # links are tested these tests can take almost an hour! num_links_to_test = 6 + # This test iterates over the fabric links on a linecard # It isolates and unisolates each fabric link. Each time the # state of the link is changed the state is checked in both @@ -77,81 +79,84 @@ def test_fabric_cli_isolate_linecards(duthosts, enum_frontend_dut_hostname): pytest_assert( originalIsolateStatus == "True" or originalIsolateStatus == "False", "Port {} CONFIG_DB initial isolateStatus is True, expected False".format(localPort)) + try: + # If the port is isolated then temporarily unisolate it + if originalIsolateStatus == "True": + cmd = "sudo config fabric port unisolate {} {}".format(localPort, asicNamespaceOption) + cmd_output = duthost.shell(cmd, module_ignore_errors=True) + stderr_output = cmd_output["stderr"] + pytest_assert( + len(stderr_output) <= 0, "command: {} failed, error: {}".format(cmd, stderr_output)) + + # Check the isolateStatus in CONFIG_DB + cmd = "sonic-db-cli {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicNamespaceOption, + localPort) + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + tokens = cmd_output[0].split() + pytest_assert( + len(tokens) > 0, + "FABRIC_PORT|Fabric{} isolateStatus not found in CONFIG_DB".format(localPort)) + isolateStatus = tokens[0] + pytest_assert( + isolateStatus == "False", + "Port {} CONFIG_DB initial isolateStatus is True, expected False".format(localPort)) + + # Check the isolateStatus in APPL_DB + cmd = "sonic-db-cli {} APPL_DB hget 'FABRIC_PORT_TABLE:Fabric{}' isolateStatus".format( + asicNamespaceOption, + localPort) + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + tokens = cmd_output[0].split() + pytest_assert( + len(tokens) > 0, "FABRIC_PORT_TABLE:Fabric{} isolateStatus not found in APPL_DB".format(localPort)) + isolateStatus = tokens[0] + pytest_assert( + isolateStatus == "False", + "Port {} APPL_DB initial isolateStatus is True, expected False".format(localPort)) - # If the port is isolated then temporarily unisolate it - if originalIsolateStatus == "True": - cmd = "sudo config fabric port unisolate {} {}".format(localPort, asicNamespaceOption) + # Isolate the port + cmd = "sudo config fabric port isolate {} {}".format(localPort, asicNamespaceOption) cmd_output = duthost.shell(cmd, module_ignore_errors=True) stderr_output = cmd_output["stderr"] pytest_assert( - len(stderr_output) <= 0, "command: {} failed, error: {}".format(cmd, stderr_output)) - - # Check the isolateStatus in CONFIG_DB - cmd = "sonic-db-cli {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicNamespaceOption, - localPort) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - tokens = cmd_output[0].split() - pytest_assert( - len(tokens) > 0, - "FABRIC_PORT|Fabric{} isolateStatus not found in CONFIG_DB".format(localPort)) - isolateStatus = tokens[0] - pytest_assert( - isolateStatus == "False", - "Port {} CONFIG_DB initial isolateStatus is True, expected False".format(localPort)) - - # Check the isolateStatus in APPL_DB - cmd = "sonic-db-cli {} APPL_DB hget 'FABRIC_PORT_TABLE:Fabric{}' isolateStatus".format(asicNamespaceOption, - localPort) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - tokens = cmd_output[0].split() - pytest_assert( - len(tokens) > 0, "FABRIC_PORT_TABLE:Fabric{} isolateStatus not found in APPL_DB".format(localPort)) - isolateStatus = tokens[0] - pytest_assert( - isolateStatus == "False", - "Port {} APPL_DB initial isolateStatus is True, expected False".format(localPort)) - - # Isolate the port - cmd = "sudo config fabric port isolate {} {}".format(localPort, asicNamespaceOption) - cmd_output = duthost.shell(cmd, module_ignore_errors=True) - stderr_output = cmd_output["stderr"] - pytest_assert( - len(stderr_output) <= 0, "command: {} failed, error: {}".format(cmd, stderr_output)) - - # Check the isolateStatus in CONFIG_DB - cmd = "sonic-db-cli {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicNamespaceOption, - localPort) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - tokens = cmd_output[0].split() - pytest_assert( - len(tokens) > 0, - "FABRIC_PORT|Fabric{} isolateStatus not found in CONFIG_DB".format(localPort)) - isolateStatus = tokens[0] - pytest_assert( - isolateStatus == "True", - "Port {} CONFIG_DB initial isolateStatus is True, expected False".format(localPort)) + len(stderr_output) <= 0, "command: {} failed, error: {}".format(cmd, stderr_output)) - # Check the isolateStatus in APPL_DB - cmd = "sonic-db-cli {} APPL_DB hget 'FABRIC_PORT_TABLE:Fabric{}' isolateStatus".format(asicNamespaceOption, + # Check the isolateStatus in CONFIG_DB + cmd = "sonic-db-cli {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicNamespaceOption, localPort) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - tokens = cmd_output[0].split() - pytest_assert( - len(tokens) > 0, - "FABRIC_PORT_TABLE:Fabric{} isolateStatus not found in APPL_DB".format(localPort)) - isolateStatus = tokens[0] - pytest_assert( - isolateStatus == "True", - "Port {} APPL_DB initial isolateStatus is True, expected False".format(localPort)) - - # If the port was originally not isolsated then restore it - if originalIsolateStatus == "False": - cmd = "sudo config fabric port unisolate {} {}".format(localPort, asicNamespaceOption) - cmd_output = duthost.shell(cmd, module_ignore_errors=True) - stderr_output = cmd_output["stderr"] + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + tokens = cmd_output[0].split() + pytest_assert( + len(tokens) > 0, + "FABRIC_PORT|Fabric{} isolateStatus not found in CONFIG_DB".format(localPort)) + isolateStatus = tokens[0] + pytest_assert( + isolateStatus == "True", + "Port {} CONFIG_DB initial isolateStatus is True, expected False".format(localPort)) + + # Check the isolateStatus in APPL_DB + cmd = "sonic-db-cli {} APPL_DB hget 'FABRIC_PORT_TABLE:Fabric{}' isolateStatus".format( + asicNamespaceOption, + localPort) + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + tokens = cmd_output[0].split() + pytest_assert( + len(tokens) > 0, + "FABRIC_PORT_TABLE:Fabric{} isolateStatus not found in APPL_DB".format(localPort)) + isolateStatus = tokens[0] pytest_assert( - len(stderr_output) <= 0, - "command: {} failed, error: {}".format(cmd, stderr_output)) + isolateStatus == "True", + "Port {} APPL_DB initial isolateStatus is True, expected False".format(localPort)) + + finally: + # If the port was originally not isolsated then restore it + if originalIsolateStatus == "False": + cmd = "sudo config fabric port unisolate {} {}".format(localPort, asicNamespaceOption) + cmd_output = duthost.shell(cmd, module_ignore_errors=True) + stderr_output = cmd_output["stderr"] + pytest_assert( + len(stderr_output) <= 0, + "command: {} failed, error: {}".format(cmd, stderr_output)) # This test iterates over the fabric links on each asic @@ -172,122 +177,127 @@ def test_fabric_cli_isolate_supervisor(duthosts, enum_supervisor_dut_hostname): num_asics = duthost.num_asics() logger.info("num_asics: {}".format(num_asics)) for asic in range(num_asics): - allPortsList = [] - portList = [] - asicName = "asic{}".format(asic) - logger.info(asicName) - - # Create list of ports - cmd = "show fabric reachability -n asic{}".format(asic) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - for line in cmd_output: - if not line: - continue - tokens = line.split() - # (localPort, remoteModule, remotePort, status) - if not tokens[0].isdigit(): - continue - localPort = tokens[0] - allPortsList.append(localPort) - - # To test a few of the links - portList = [] - while len(portList) < num_links_to_test: - randomPort = random.choice(allPortsList) - if randomPort not in portList: - portList.append(randomPort) - - # Test each fabric link - for localPort in portList: - logger.info("local port {}".format(localPort)) - # continue - - # Get the current isolation status of the port - cmd = "sonic-db-cli -n {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicName, localPort) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - tokens = cmd_output[0].split() - originalIsolateStatus = tokens[0] - pytest_assert( - originalIsolateStatus == "True" or originalIsolateStatus == "False", - "Port {} CONFIG_DB initial isolateStatus is True, expected False".format(localPort)) - logger.debug("originalIsolateStatus: {}".format(originalIsolateStatus)) - - # If the port is isolated then temporarily unisolate it - if originalIsolateStatus == "True": - cmd = "sudo config fabric port unisolate {} -n {}".format(localPort, asicName) - cmd_output = duthost.shell(cmd, module_ignore_errors=True) - stderr_output = cmd_output["stderr"] - pytest_assert( - len(stderr_output) <= 0, - "command: {} failed, error: {}".format(cmd, stderr_output)) - - # Check the isolateStatus in CONFIG_DB - cmd = "sonic-db-cli -n {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicName, localPort) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - tokens = cmd_output[0].split() - originalIsolateStatus = tokens[0] - pytest_assert( - len(tokens) > 0, - "FABRIC_PORT|Fabric{} isolateStatus not found in CONFIG_DB, {} ".format(localPort, asicName)) - isolateStatus = tokens[0] - pytest_assert( - isolateStatus == "False", - "Port {} CONFIG_DB initial isolateStatus is '{}', expected False".format(localPort, isolateStatus)) - - # Check the isolateStatus in APPL_DB - cmd = "sonic-db-cli -n {} APPL_DB hget 'FABRIC_PORT_TABLE:Fabric{}' isolateStatus".format(asicName, - localPort) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - tokens = cmd_output[0].split() - originalIsolateStatus = tokens[0] - pytest_assert( - len(tokens) > 0, - "FABRIC_PORT_TABLE:Fabric{} isolateStatus not found in APPL_DB, {} ".format(localPort, asicName)) - isolateStatus = tokens[0] - pytest_assert( - isolateStatus == "False", - "Port {} APPL_DB initial isolateStatus is '{}', expected False".format(localPort, isolateStatus)) - - # Isolate the port - cmd = "sudo config fabric port isolate {} -n {}".format(localPort, asicName) - cmd_output = duthost.shell(cmd, module_ignore_errors=True) - stderr_output = cmd_output["stderr"] - pytest_assert( - len(stderr_output) <= 0, - "command: {} failed, error: {}".format(cmd, stderr_output)) - - # Check the isolateStatus in CONFIG_DB - cmd = "sonic-db-cli -n {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicName, localPort) + if asic in duthost.facts['asics_present']: + allPortsList = [] + portList = [] + asicName = "asic{}".format(asic) + logger.info(asicName) + + # Create list of ports + cmd = "show fabric reachability -n asic{}".format(asic) cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - tokens = cmd_output[0].split() - originalIsolateStatus = tokens[0] - pytest_assert( - len(tokens) > 0, - "FABRIC_PORT|Fabric{} isolateStatus not found in CONFIG_DB, {} ".format(localPort, asicName)) - isolateStatus = tokens[0] - pytest_assert( - isolateStatus == "True", - "Port {} CONFIG_DB initial isolateStatus is '{}', expected False".format(localPort, isolateStatus)) - - # Check the isolateStatus in APPL_DB - cmd = "sonic-db-cli -n {} APPL_DB hget 'FABRIC_PORT_TABLE:Fabric{}' isolateStatus".format(asicName, + for line in cmd_output: + if not line: + continue + tokens = line.split() + # (localPort, remoteModule, remotePort, status) + if not tokens[0].isdigit(): + continue + localPort = tokens[0] + allPortsList.append(localPort) + + # To test a few of the links + portList = [] + while len(portList) < num_links_to_test: + randomPort = random.choice(allPortsList) + if randomPort not in portList: + portList.append(randomPort) + + # Test each fabric link + for localPort in portList: + logger.info("local port {}".format(localPort)) + # continue + + # Get the current isolation status of the port + cmd = "sonic-db-cli -n {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicName, localPort) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - tokens = cmd_output[0].split() - originalIsolateStatus = tokens[0] - pytest_assert( - len(tokens) > 0, - "FABRIC_PORT_TABLE:Fabric{} isolateStatus not found in APPL_DB, {} ".format(localPort, asicName)) - isolateStatus = tokens[0] - pytest_assert( - isolateStatus == "True", - "Port {} APPL_DB initial isolateStatus is '{}', expected False".format(localPort, isolateStatus)) - - # If the port was originally not isolsated then restore it - if originalIsolateStatus == "False": - cmd = "sudo config fabric port unisolate {} -n {}".format(localPort, asicName) - cmd_output = duthost.shell(cmd, module_ignore_errors=True) - stderr_output = cmd_output["stderr"] + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + tokens = cmd_output[0].split() + originalIsolateStatus = tokens[0] pytest_assert( - len(stderr_output) <= 0, - "command: {} failed, error: {}".format(cmd, stderr_output)) + originalIsolateStatus == "True" or originalIsolateStatus == "False", + "Port {} CONFIG_DB initial isolateStatus is True, expected False".format(localPort)) + logger.debug("originalIsolateStatus: {}".format(originalIsolateStatus)) + try: + # If the port is isolated then temporarily unisolate it + if originalIsolateStatus == "True": + cmd = "sudo config fabric port unisolate {} -n {}".format(localPort, asicName) + cmd_output = duthost.shell(cmd, module_ignore_errors=True) + stderr_output = cmd_output["stderr"] + pytest_assert( + len(stderr_output) <= 0, + "command: {} failed, error: {}".format(cmd, stderr_output)) + + # Check the isolateStatus in CONFIG_DB + cmd = "sonic-db-cli -n {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicName, + localPort) + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + tokens = cmd_output[0].split() + pytest_assert( + len(tokens) > 0, + "FABRIC_PORT|Fabric{} isolateStatus not found in CONFIG_DB, {} ".format(localPort, asicName)) + isolateStatus = tokens[0] + pytest_assert( + isolateStatus == "False", + "Port {} CONFIG_DB initial isolateStatus is '{}', expected False".format(localPort, + isolateStatus)) + + # Check the isolateStatus in APPL_DB + cmd = "sonic-db-cli -n {} APPL_DB hget 'FABRIC_PORT_TABLE:Fabric{}' isolateStatus".format(asicName, + localPort) + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + tokens = cmd_output[0].split() + pytest_assert( + len(tokens) > 0, + "FABRIC_PORT_TABLE:Fabric{} isolateStatus not found in APPL_DB, {} ".format(localPort, + asicName)) + isolateStatus = tokens[0] + pytest_assert( + isolateStatus == "False", + "Port {} APPL_DB initial isolateStatus is '{}', expected False".format(localPort, + isolateStatus)) + + # Isolate the port + cmd = "sudo config fabric port isolate {} -n {}".format(localPort, asicName) + cmd_output = duthost.shell(cmd, module_ignore_errors=True) + stderr_output = cmd_output["stderr"] + pytest_assert( + len(stderr_output) <= 0, + "command: {} failed, error: {}".format(cmd, stderr_output)) + + # Check the isolateStatus in CONFIG_DB + cmd = "sonic-db-cli -n {} CONFIG_DB hget 'FABRIC_PORT|Fabric{}' isolateStatus".format(asicName, + localPort) + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + tokens = cmd_output[0].split() + pytest_assert(len(tokens) > 0, + "FABRIC_PORT|Fabric{} isolateStatus not found in CONFIG_DB, {} ".format(localPort, + asicName)) + isolateStatus = tokens[0] + pytest_assert( + isolateStatus == "True", + "Port {} CONFIG_DB initial isolateStatus is '{}', expected True".format(localPort, + isolateStatus)) + + # Check the isolateStatus in APPL_DB + cmd = "sonic-db-cli -n {} APPL_DB hget 'FABRIC_PORT_TABLE:Fabric{}' isolateStatus".format(asicName, + localPort) + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + tokens = cmd_output[0].split() + pytest_assert( + len(tokens) > 0, + "FABRIC_PORT_TABLE:Fabric{} isolateStatus not found in APPL_DB, {} ".format(localPort, + asicName)) + isolateStatus = tokens[0] + pytest_assert(isolateStatus == "True", + "Port {} APPL_DB initial isolateStatus is '{}', expected True".format(localPort, + isolateStatus)) + finally: + # If the port was originally not isolsated then restore it + if originalIsolateStatus == "False": + cmd = "sudo config fabric port unisolate {} -n {}".format(localPort, asicName) + cmd_output = duthost.shell(cmd, module_ignore_errors=True) + stderr_output = cmd_output["stderr"] + pytest_assert( + len(stderr_output) <= 0, + "command: {} failed, error: {}".format(cmd, stderr_output)) diff --git a/tests/voq/test_fabric_reach.py b/tests/voq/test_fabric_reach.py index 215a91780d6..67a0f6d19e8 100644 --- a/tests/voq/test_fabric_reach.py +++ b/tests/voq/test_fabric_reach.py @@ -175,34 +175,37 @@ def test_fabric_reach_supervisor(duthosts, enum_supervisor_dut_hostname, refData # supReferenceData has the expected data duthost = duthosts[enum_supervisor_dut_hostname] logger.info("duthost: {}".format(duthost.hostname)) + if not supReferenceData: + supRefData(duthosts) num_asics = duthost.num_asics() logger.info("num_asics: {}".format(num_asics)) for asic in range(num_asics): - asicName = "asic{}".format(asic) - logger.info(asicName) - cmd = "show fabric reachability -n asic{}".format(asic) - cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") - asicReferenceData = supReferenceData[asicName] - for line in cmd_output: - if not line: - continue - tokens = line.split() - if not tokens[0].isdigit(): - continue - localPortName = tokens[0] - remoteModule = int(tokens[1]) - remotePort = int(tokens[2]) - if remoteModule not in linecardModule: - logger.info("The linecard is not inserted or down.") - continue - pytest_assert(localPortName in asicReferenceData, - "Reference port data for {} not found!".format(localPortName)) - referencePortData = asicReferenceData[localPortName] - referenceRemoteModule = referencePortData['peer mod'] - referenceRemotePort = referencePortData['peer lk'] - pytest_assert(remoteModule == referenceRemoteModule, - "Remote module mismatch for asic {}, port {}" - .format(asicName, localPortName)) - pytest_assert(remotePort == referenceRemotePort, - "Remote port mismatch for asic {}, port {}" - .format(asicName, localPortName)) + if asic in duthost.facts['asics_present']: + asicName = "asic{}".format(asic) + logger.info(asicName) + cmd = "show fabric reachability -n asic{}".format(asic) + cmd_output = duthost.shell(cmd, module_ignore_errors=True)["stdout"].split("\n") + asicReferenceData = supReferenceData[asicName] + for line in cmd_output: + if not line: + continue + tokens = line.split() + if not tokens[0].isdigit(): + continue + localPortName = tokens[0] + remoteModule = int(tokens[1]) + remotePort = int(tokens[2]) + if remoteModule not in linecardModule: + logger.info("The linecard is not inserted or down.") + continue + pytest_assert(localPortName in asicReferenceData, + "Reference port data for {} not found!".format(localPortName)) + referencePortData = asicReferenceData[localPortName] + referenceRemoteModule = referencePortData['peer mod'] + referenceRemotePort = referencePortData['peer lk'] + pytest_assert(remoteModule == referenceRemoteModule, + "Remote module mismatch for asic {}, port {}" + .format(asicName, localPortName)) + pytest_assert(remotePort == referenceRemotePort, + "Remote port mismatch for asic {}, port {}" + .format(asicName, localPortName))