tests(iroh): Packet loss is expected with socket rebinding (#3001)

## Description

This allows packet loss when rebinding sockets.

## Breaking Changes

<!-- Optional, if there are any breaking changes document them,
including how to migrate older code. -->

## Notes & open questions

I think it would also be reasonable to use e.g.:

``` rust
enum ExpectedLoss {
    Low,
    Medium,
}
```

And then accept like 50 lost packets in Medium.


## Change checklist

- [x] Self-review.
- [x] Documentation updates following the [style
guide](https://rust-lang.github.io/rfcs/1574-more-api-documentation-conventions.html#appendix-a-full-conventions-text),
if relevant.
- [x] Tests if relevant.
- [x] All breaking changes documented.

iroh/src/magicsock.rs

@@ -2995,7 +2995,7 @@ mod tests {
     }
 
     #[instrument(skip_all, fields(me = %ep.endpoint.node_id().fmt_short()))]
-    async fn echo_receiver(ep: MagicStack) -> Result<()> {
+    async fn echo_receiver(ep: MagicStack, loss: ExpectedLoss) -> Result<()> {
         info!("accepting conn");
         let conn = ep.endpoint.accept().await.expect("no conn");
 
@@ -3026,10 +3026,12 @@ mod tests {
         let stats = conn.stats();
         info!("stats: {:#?}", stats);
         // TODO: ensure panics in this function are reported ok
-        assert!(
-            stats.path.lost_packets < 10,
-            "[receiver] should not loose many packets",
-        );
+        if matches!(loss, ExpectedLoss::AlmostNone) {
+            assert!(
+                stats.path.lost_packets < 10,
+                "[receiver] should not loose many packets",
+            );
+        }
 
         info!("close");
         conn.close(0u32.into(), b"done");
@@ -3040,7 +3042,12 @@ mod tests {
     }
 
     #[instrument(skip_all, fields(me = %ep.endpoint.node_id().fmt_short()))]
-    async fn echo_sender(ep: MagicStack, dest_id: PublicKey, msg: &[u8]) -> Result<()> {
+    async fn echo_sender(
+        ep: MagicStack,
+        dest_id: PublicKey,
+        msg: &[u8],
+        loss: ExpectedLoss,
+    ) -> Result<()> {
         info!("connecting to {}", dest_id.fmt_short());
         let dest = NodeAddr::new(dest_id);
         let conn = ep
@@ -3071,10 +3078,12 @@ mod tests {
 
         let stats = conn.stats();
         info!("stats: {:#?}", stats);
-        assert!(
-            stats.path.lost_packets < 10,
-            "[sender] should not loose many packets",
-        );
+        if matches!(loss, ExpectedLoss::AlmostNone) {
+            assert!(
+                stats.path.lost_packets < 10,
+                "[sender] should not loose many packets",
+            );
+        }
 
         info!("close");
         conn.close(0u32.into(), b"done");
@@ -3083,14 +3092,25 @@ mod tests {
         Ok(())
     }
 
+    #[derive(Debug, Copy, Clone)]
+    enum ExpectedLoss {
+        AlmostNone,
+        YeahSure,
+    }
+
     /// Runs a roundtrip between the [`echo_sender`] and [`echo_receiver`].
-    async fn run_roundtrip(sender: MagicStack, receiver: MagicStack, payload: &[u8]) {
+    async fn run_roundtrip(
+        sender: MagicStack,
+        receiver: MagicStack,
+        payload: &[u8],
+        loss: ExpectedLoss,
+    ) {
         let send_node_id = sender.endpoint.node_id();
         let recv_node_id = receiver.endpoint.node_id();
         info!("\nroundtrip: {send_node_id:#} -> {recv_node_id:#}");
 
-        let receiver_task = tokio::spawn(echo_receiver(receiver));
-        let sender_res = echo_sender(sender, recv_node_id, payload).await;
+        let receiver_task = tokio::spawn(echo_receiver(receiver, loss));
+        let sender_res = echo_sender(sender, recv_node_id, payload, loss).await;
         let sender_is_err = match sender_res {
             Ok(()) => false,
             Err(err) => {
@@ -3129,14 +3149,26 @@ mod tests {
 
         for i in 0..5 {
             info!("\n-- round {i}");
-            run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await;
-            run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await;
+            run_roundtrip(
+                m1.clone(),
+                m2.clone(),
+                b"hello m1",
+                ExpectedLoss::AlmostNone,
+            )
+            .await;
+            run_roundtrip(
+                m2.clone(),
+                m1.clone(),
+                b"hello m2",
+                ExpectedLoss::AlmostNone,
+            )
+            .await;
 
             info!("\n-- larger data");
             let mut data = vec![0u8; 10 * 1024];
             rand::thread_rng().fill_bytes(&mut data);
-            run_roundtrip(m1.clone(), m2.clone(), &data).await;
-            run_roundtrip(m2.clone(), m1.clone(), &data).await;
+            run_roundtrip(m1.clone(), m2.clone(), &data, ExpectedLoss::AlmostNone).await;
+            run_roundtrip(m2.clone(), m1.clone(), &data, ExpectedLoss::AlmostNone).await;
         }
 
         Ok(())
@@ -3223,14 +3255,14 @@ mod tests {
 
         for i in 0..rounds {
             println!("-- [m1 changes] round {}", i + 1);
-            run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await;
-            run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await;
+            run_roundtrip(m1.clone(), m2.clone(), b"hello m1", ExpectedLoss::YeahSure).await;
+            run_roundtrip(m2.clone(), m1.clone(), b"hello m2", ExpectedLoss::YeahSure).await;
 
             println!("-- [m1 changes] larger data");
             let mut data = vec![0u8; 10 * 1024];
             rand::thread_rng().fill_bytes(&mut data);
-            run_roundtrip(m1.clone(), m2.clone(), &data).await;
-            run_roundtrip(m2.clone(), m1.clone(), &data).await;
+            run_roundtrip(m1.clone(), m2.clone(), &data, ExpectedLoss::YeahSure).await;
+            run_roundtrip(m2.clone(), m1.clone(), &data, ExpectedLoss::YeahSure).await;
         }
 
         std::mem::drop(m1_network_change_guard);
@@ -3252,14 +3284,14 @@ mod tests {
 
         for i in 0..rounds {
             println!("-- [m2 changes] round {}", i + 1);
-            run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await;
-            run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await;
+            run_roundtrip(m1.clone(), m2.clone(), b"hello m1", ExpectedLoss::YeahSure).await;
+            run_roundtrip(m2.clone(), m1.clone(), b"hello m2", ExpectedLoss::YeahSure).await;
 
             println!("-- [m2 changes] larger data");
             let mut data = vec![0u8; 10 * 1024];
             rand::thread_rng().fill_bytes(&mut data);
-            run_roundtrip(m1.clone(), m2.clone(), &data).await;
-            run_roundtrip(m2.clone(), m1.clone(), &data).await;
+            run_roundtrip(m1.clone(), m2.clone(), &data, ExpectedLoss::YeahSure).await;
+            run_roundtrip(m2.clone(), m1.clone(), &data, ExpectedLoss::YeahSure).await;
         }
 
         std::mem::drop(m2_network_change_guard);
@@ -3282,14 +3314,14 @@ mod tests {
 
         for i in 0..rounds {
             println!("-- [m1 & m2 changes] round {}", i + 1);
-            run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await;
-            run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await;
+            run_roundtrip(m1.clone(), m2.clone(), b"hello m1", ExpectedLoss::YeahSure).await;
+            run_roundtrip(m2.clone(), m1.clone(), b"hello m2", ExpectedLoss::YeahSure).await;
 
             println!("-- [m1 & m2 changes] larger data");
             let mut data = vec![0u8; 10 * 1024];
             rand::thread_rng().fill_bytes(&mut data);
-            run_roundtrip(m1.clone(), m2.clone(), &data).await;
-            run_roundtrip(m2.clone(), m1.clone(), &data).await;
+            run_roundtrip(m1.clone(), m2.clone(), &data, ExpectedLoss::YeahSure).await;
+            run_roundtrip(m2.clone(), m1.clone(), &data, ExpectedLoss::YeahSure).await;
         }
 
         std::mem::drop(m1_m2_network_change_guard);