[test] fix potential backbone name conflict when running cert suite (#…

…10596)

### Background

#10550 introduced a new
way to support multiple backbone nework in otbr docker test. Though it
works good while running a single test, a bug exists when running
cert-suite, which runs a batch of tests in parallel.

cert-suite allocates the name of the backbone interfaces dynamically
by setting env PORT_OFFSET for each test, so there is potentially
conflict exists if we hard code the `backbone_network` name in
TOPOLOGY. This PR is targeting to fix this potential naming conflict.

### Fix

We fix it by assigning a number for `backbone_network_id` in each BR
definition in TOPOLOGY, instead of setting a fixed `backbone network`
name. The final backbone network name is decided by both `PORT_OFFSET`
env and the number of `backbone_network` (in
`backbone{PORT_OFFSET}.{backbone_network}` format)

For example, if `PORT_OFFSET` is 0 and `backbone_network_id` is 1,
then backbone network name will be `backbone0.1`. For the tests that
only use one backbone network and the `backbone_network_id` is not
given, the backbone network name is by default
`backbone{PORT_OFFSET}.0`.

### New test case format
```
class NewTestCase(thread_cert.TestCase):
    ...
    BR = 1
    ...
    TOPOLOGY = {
        BR: {
            ...
            'is_otbr': True,
            'backbone_network_id': <backbone-id>,
            ...
        }
        ...
    }
    ...
```

`<backbone-id>` is any integer from 0, for each BR inside a single
test, if `<backbone-id>` is different, the BR use different backbone
interfaces; the same `<backbone-id>` inside a single test case means
the same backbone network interface.

`'backbone_network_id': <backbone-id>` is optional for single backbone
test cases, when it's not given while defining a otbr node, the
backbone is default as `backbone{PORT_OFFSET}.0`.

For developers, if you are defining a new test which has multiple
backbone interfaces, please ensure `backbone_network_id` is explicitly
defined in each BR, otherwize an error is reported.

tests/scripts/thread-cert/border_router/test_multi_ail.py

@@ -42,7 +42,7 @@ class TwoBorderRoutersOnTwoInfrastructures(thread_cert.TestCase):
 
  Topology:
 
- -------(backbone0)-------- | ---------(backbone1)-------
+ -------(backbone0.0)-------- | ---------(backbone0.1)-------
  | |
  BR1 (Leader) .............. BR2 (Router)
 
@@ -55,14 +55,14 @@ class TwoBorderRoutersOnTwoInfrastructures(thread_cert.TestCase):
  TOPOLOGY = {
  BR1: {
  'name': 'BR_1',
- 'backbone_network': 'backbone0',
+ 'backbone_network_id': 0,
  'allowlist': [BR2],
  'is_otbr': True,
  'version': '1.3',
  },
  BR2: {
  'name': 'BR_2',
- 'backbone_network': 'backbone1',
+ 'backbone_network_id': 1,
  'allowlist': [BR1],
  'is_otbr': True,
  'version': '1.3',

tests/scripts/thread-cert/config.py

@@ -64,11 +64,11 @@
 DOMAIN_PREFIX_ALTER = 'fd00:7d04:7d04:7d04::/64'
 
 PORT_OFFSET = int(os.getenv('PORT_OFFSET', '0'))
-BACKBONE_IPV6_ADDR_START_BASE = 0x9100
-BACKBONE_IPV6_ADDR_START = BACKBONE_IPV6_ADDR_START_BASE + PORT_OFFSET
-BACKBONE_PREFIX = f'{BACKBONE_IPV6_ADDR_START:04x}::/64'
-BACKBONE_PREFIX_REGEX_PATTERN = f'^{BACKBONE_IPV6_ADDR_START:04x}:'
+BACKBONE_IPV6_ADDR_START = f'{0x9100 + PORT_OFFSET:04x}'
+BACKBONE_PREFIX = f'{BACKBONE_IPV6_ADDR_START}::/64'
+BACKBONE_PREFIX_REGEX_PATTERN = f'^{BACKBONE_IPV6_ADDR_START}:'
 BACKBONE_DOCKER_NETWORK_NAME = f'backbone{PORT_OFFSET}'
+BACKBONE_DOCKER_NETWORK_DEFAULT_ID = 0
 BACKBONE_IFNAME = 'eth0'
 THREAD_IFNAME = 'wpan0'
 

tests/scripts/thread-cert/node.py

@@ -66,6 +66,8 @@ class OtbrDocker:
 
  def __init__(self, nodeid: int, backbone_network: str, **kwargs):
  self.verbose = int(float(os.getenv('VERBOSE', 0)))
+
+ assert backbone_network is not None
  self.backbone_network = backbone_network
  try:
  self._docker_name = config.OTBR_DOCKER_NAME_PREFIX + str(nodeid)

tests/scripts/thread-cert/run_cert_suite.py

@@ -71,7 +71,7 @@ def run_cert(job_id: int, port_offset: int, script: str, run_directory: str):
  env['PYTHONPATH'] = os.path.dirname(os.path.abspath(__file__))
 
  try:
- print(f'Running {test_name}')
+ print(f'Running PORT_OFFSET={port_offset} {test_name}')
  with open(logfile, 'wt') as output:
  abs_script = os.path.abspath(script)
  subprocess.check_call(abs_script,

tests/scripts/thread-cert/thread_cert.py

@@ -111,6 +111,10 @@ def __init__(self, *args, **kwargs):
  self._do_packet_verification = PACKET_VERIFICATION and hasattr(self, 'verify') \
  and self.PACKET_VERIFICATION == PACKET_VERIFICATION
 
+ # store all the backbone network names that are used in the test case,
+ # it keeps empty when there's no backbone traffic in the test (no otbr or host nodes)
+ self._backbone_network_names = []
+
  def skipTest(self, reason: Any) -> None:
  self._testSkipped = True
  super(TestCase, self).skipTest(reason)
@@ -153,7 +157,11 @@ def _setUp(self):
  params = self._parse_params(params)
  initial_topology[i] = params
 
+ backbone_network_name = self._construct_backbone_network_name(params.get('backbone_network_id')) \
+ if self._has_backbone_traffic() else None
+
  logging.info("Creating node %d: %r", i, params)
+ logging.info("Backbone network: %s", backbone_network_name)
 
  if params['is_otbr']:
  nodeclass = OtbrNode
@@ -168,7 +176,7 @@ def _setUp(self):
  name=params.get('name'),
  version=params['version'],
  is_bbr=params['is_bbr'],
- backbone_network=params['backbone_network'])
+ backbone_network=backbone_network_name)
  if 'boot_delay' in params:
  self.simulator.go(params['boot_delay'])
 
@@ -465,6 +473,18 @@ def _collect_test_info_after_setup(self):
  ethaddr = node.get_ether_mac()
  test_info['ethaddrs'][i] = EthAddr(ethaddr).format_octets()
 
+ def _construct_backbone_network_name(self, backbone_network_id) -> str:
+ """
+ Construct the name of the backbone network based on the given backbone network id from TOPOLOGY. If the
+ backbone_network_id is not defined in TOPOLOGY, use the default backbone network id.
+ """
+ id = backbone_network_id if backbone_network_id is not None else config.BACKBONE_DOCKER_NETWORK_DEFAULT_ID
+ backbone_name = f'{config.BACKBONE_DOCKER_NETWORK_NAME}.{id}'
+
+ assert backbone_name in self._backbone_network_names
+
+ return backbone_name
+
  def _output_test_info(self):
  """
  Output test info to json file after tearDown
@@ -500,9 +520,6 @@ def _parse_params(self, params: Optional[dict]) -> dict:
  assert params.get('version', '') == '', params
  params['version'] = ''
 
- # set default backbone network for bbr/otbr/host if not specified
- params.setdefault('backbone_network', config.BACKBONE_DOCKER_NETWORK_NAME)
-
  # use 1.3 node for 1.2 tests
  if params.get('version') == '1.2':
  params['version'] = '1.3'
@@ -527,35 +544,45 @@ def _has_backbone_traffic(self):
 
  def _prepare_backbone_network(self):
  """
- Prepares one or multiple backbone network(s).
+ Creates one or more backbone networks (Docker bridge networks) based on the TOPOLOGY definition.
 
- This method creates the backbone network based on the `backbone` values defined in the TOPOLOGY in each test.
- If no backbone values are defined, it sets the default backbone network as BACKBONE_DOCKER_NETWORK_NAME
- from the config module.
+ * If `backbone_network_id` is defined in the TOPOLOGY:
+ * Network name: `backbone{PORT_OFFSET}.{backbone_network_id}` (e.g., "backbone0.0", "backbone0.1")
+ * Network prefix: `backbone{PORT_OFFSET}:{backbone_network_id}::/64` (e.g., "9100:0::/64", "9100:1::/64")
+
+ * If `backbone_network_id` is undefined:
+ * Network name: `backbone{PORT_OFFSET}.0` (e.g., "backbone0.0")
+ * Network prefix: `backbone{PORT_OFFSET}::/64` (e.g., "9100::/64")
  """
- # Use backbone_set to store all the backbone values in TOPOLOGY
- backbone_set = set()
+ # Create backbone_set to store all the backbone_ids by parsing TOPOLOGY.
+ backbone_id_set = set()
  for node in self.TOPOLOGY:
- backbone = self.TOPOLOGY[node].get('backbone_network')
- if backbone:
- backbone_set.add(backbone)
+ id = self.TOPOLOGY[node].get('backbone_network_id')
+ if id is not None:
+ backbone_id_set.add(id)
 
- # Set default backbone network if backbone_set is empty
- if not backbone_set:
- backbone_set.add(config.BACKBONE_DOCKER_NETWORK_NAME)
+ # Add default backbone network id if backbone_set is empty
+ if not backbone_id_set:
+ backbone_id_set.add(config.BACKBONE_DOCKER_NETWORK_DEFAULT_ID)
 
  # Iterate over the backbone_set and create backbone network(s)
- for offset, backbone in enumerate(backbone_set, start=PORT_OFFSET):
- backbone_prefix = f'{config.BACKBONE_IPV6_ADDR_START_BASE + offset:04x}::/64'
+ for id in backbone_id_set:
+ backbone = f'{config.BACKBONE_DOCKER_NETWORK_NAME}.{id}'
+ backbone_prefix = f'{config.BACKBONE_IPV6_ADDR_START}:{id}::/64'
+ self._backbone_network_names.append(backbone)
  self.assure_run_ok(
  f'docker network create --driver bridge --ipv6 --subnet {backbone_prefix} -o "com.docker.network.bridge.name"="{backbone}" {backbone} || true',
  shell=True)
 
  def _remove_backbone_network(self):
- network_name = config.BACKBONE_DOCKER_NETWORK_NAME
- self.assure_run_ok(f'docker network rm {network_name}', shell=True)
+ for network_name in self._backbone_network_names:
+  self.assure_run_ok(f'docker network rm {network_name}', shell=True)
 
  def _start_backbone_sniffer(self):
+ assert self._backbone_network_names, 'Internal Error: self._backbone_network_names is empty'
+ # TODO: support sniffer on multiple backbone networks
+ sniffer_interface = self._backbone_network_names[0]
+
  # don't know why but I have to create the empty bbr.pcap first, otherwise tshark won't work
  # self.assure_run_ok("truncate --size 0 bbr.pcap && chmod 664 bbr.pcap", shell=True)
  pcap_file = self._get_backbone_pcap_filename()
@@ -565,12 +592,13 @@ def _start_backbone_sniffer(self):
  pass
 
  dumpcap = pvutils.which_dumpcap()
- self._dumpcap_proc = subprocess.Popen([dumpcap, '-i', config.BACKBONE_DOCKER_NETWORK_NAME, '-w', pcap_file],
+ self._dumpcap_proc = subprocess.Popen([dumpcap, '-i', sniffer_interface, '-w', pcap_file],
  stdout=sys.stdout,
  stderr=sys.stderr)
  time.sleep(0.2)
  assert self._dumpcap_proc.poll() is None, 'tshark terminated unexpectedly'
- logging.info('Backbone sniffer launched successfully: pid=%s', self._dumpcap_proc.pid)
+ logging.info('Backbone sniffer launched successfully on interface %s, pid=%s', sniffer_interface,
+ self._dumpcap_proc.pid)
  os.chmod(pcap_file, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
 
  def _get_backbone_pcap_filename(self):