Possibility for datagrams to be dropped when not able to send

docs/api/DatagramSend.md

@@ -21,7 +21,36 @@ QUIC_STATUS
 
 # Parameters
 
-**TODO**
+`Connection`
+
+The current established connection.
+
+`Buffers`
+
+An array of `QUIC_BUFFER` structs that each contain a pointer and length to app data to send on the stream. This may be `NULL` **only** if `BufferCount` is zero.
+
+`BufferCount`
+
+The number of `QUIC_BUFFER` structs in the `Buffers` array. This may be zero.
+
+`Flags`
+
+The set of flags that controls the behavior of `DatagramSend`:
+
+Value | Meaning
+--- | ---
+**QUIC_SEND_FLAG_NONE**<br>0 | No special behavior. Data is not allowed in 0-RTT by default.
+**QUIC_SEND_FLAG_ALLOW_0_RTT**<br>1 | Indicates that the data is allowed to be sent in 0-RTT (if available). Makes no guarantee the data will be sent in 0-RTT. Additionally, even if 0-RTT keys are available the data may end up being sent in 1-RTT for multiple reasons.
+**QUIC_SEND_FLAG_START**<br>2 | **Unused and ignored** for `DatagramSend`
+**QUIC_SEND_FLAG_FIN**<br>4 | **Unused and ignored** for `DatagramSend`
+**QUIC_SEND_FLAG_DGRAM_PRIORITY**<br>8 | Sets a priority to ensure a datagram is sent before others.
+**QUIC_SEND_FLAG_DELAY_SEND**<br>16 | **Unused and ignored** for `DatagramSend`
+**QUIC_SEND_FLAG_CANCEL_ON_LOSS**<br>32 | **Unused and ignored** for `DatagramSend`
+**QUIC_SEND_FLAG_CANCEL_ON_BLOCKED**<br>64 | Allows MsQuic to drop frames when all the data that could be sent has been flushed out, but there are still some frames remaining in the queue.
+
+`ClientSendContext`
+
+The app context pointer (possibly null) to be associated with the send.
 
 # Return Value
 

docs/api/StreamSend.md

@@ -46,6 +46,7 @@ Value | Meaning
 **QUIC_SEND_FLAG_DGRAM_PRIORITY**<br>8 | **Unused and ignored** for `StreamSend`
 **QUIC_SEND_FLAG_DELAY_SEND**<br>16 | Provides a hint to MsQuic to indicate the data does not need to be sent immediately, likely because more is soon to follow.
 **QUIC_SEND_FLAG_CANCEL_ON_LOSS**<br>32 | Informs MsQuic to irreversibly mark the associated stream to be canceled when packet loss has been detected on it. I.e., all sends on a given stream are subject to this behavior from the moment the flag has been supplied for the first time. 
+**QUIC_SEND_FLAG_CANCEL_ON_BLOCKED**<br>64 | **Unused and ignored** for `StreamSend` for now
 
 `ClientSendContext`
 

src/core/datagram.c

@@ -588,3 +588,34 @@
 
     return TRUE;
 }
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+QuicDatagramCancelBlocked(
+    _In_ QUIC_CONNECTION* Connection
+    )
+{
+    QUIC_DATAGRAM* Datagram = &Connection->Datagram;
+    QUIC_SEND_REQUEST** SendQueue = &Datagram->SendQueue;
+    while (*SendQueue != NULL) {
+        if ((*SendQueue)->Flags & QUIC_SEND_FLAG_CANCEL_ON_BLOCKED) {
+            QUIC_SEND_REQUEST* SendRequest = *SendQueue;
+            if (Datagram->PrioritySendQueueTail == &SendRequest->Next) {
+                Datagram->PrioritySendQueueTail = SendQueue;
+            }
+            *SendQueue = SendRequest->Next;
+            QuicDatagramCancelSend(Connection, SendRequest);
+        } else {
+            SendQueue = &((*SendQueue)->Next);
+        }
+    }
+    Datagram->SendQueueTail = SendQueue;
+
+    if (Datagram->SendQueue != NULL) {
+        QuicSendSetSendFlag(&Connection->Send, QUIC_CONN_SEND_FLAG_DATAGRAM);
+    } else {
+        QuicSendClearSendFlag(&Connection->Send, QUIC_CONN_SEND_FLAG_DATAGRAM);
+    }
+
+    QuicDatagramValidate(Datagram);
+}

src/core/datagram.h

@@ -103,3 +103,9 @@ QuicDatagramProcessFrame(
         const uint8_t * const Buffer,
     _Inout_ uint16_t* Offset
     );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+QuicDatagramCancelBlocked(
+    _In_ QUIC_CONNECTION* Connection
+    );

src/core/send.c

@@ -1498,6 +1498,11 @@ QuicSendFlush(
         //QuicConnUpdatePeerPacketTolerance(Connection, Builder.TotalCountDatagrams);
     }
 
+    //
+    // Clears the SendQueue list of not sent packets if the flag is applied
+    //
+    QuicDatagramCancelBlocked(Connection);
+
     return Result != QUIC_SEND_INCOMPLETE;
 }
 #pragma warning(pop)

src/cs/lib/msquic_generated.cs

@@ -196,6 +196,7 @@ internal enum QUIC_SEND_FLAGS
         DELAY_SEND = 0x0010,
         CANCEL_ON_LOSS = 0x0020,
         PRIORITY_WORK = 0x0040,
+        CANCEL_ON_BLOCKED = 0x0080,
     }
 
     internal enum QUIC_DATAGRAM_SEND_STATE

src/inc/msquic.h

@@ -243,6 +243,7 @@ typedef enum QUIC_SEND_FLAGS {
     QUIC_SEND_FLAG_DELAY_SEND               = 0x0010,   // Indicates the send should be delayed because more will be queued soon.
     QUIC_SEND_FLAG_CANCEL_ON_LOSS           = 0x0020,   // Indicates that a stream is to be cancelled when packet loss is detected.
     QUIC_SEND_FLAG_PRIORITY_WORK            = 0x0040,   // Higher priority than other connection work.
+    QUIC_SEND_FLAG_CANCEL_ON_BLOCKED        = 0x0080,   // Indicates that a frame should be dropped when it can't be sent immediately.
 } QUIC_SEND_FLAGS;
 
 DEFINE_ENUM_FLAG_OPERATORS(QUIC_SEND_FLAGS)

src/plugins/trace/README.md

@@ -51,7 +51,9 @@ One of the built-in capabilities of WPA is the ability to analyze CPU trace info
 ### Linux
 Linux perf command is one of the way to collect such information.  
 ```sh
-# on Linux
+# on Linux (kernel > 5.10)
+sudo apt-get install -y linux-perf
+# on Linux (kernel <= 5.10)
 sudo apt-get install -y linux-tools-`uname -r`
 # use your own options
 perf record -a -g -F 10 -o out.perf.data

src/test/lib/DatagramTest.cpp

@@ -156,7 +156,9 @@ QuicTestDatagramSend(
     TEST_TRUE(ServerConfiguration.IsValid());
 
     uint8_t RawBuffer[] = "datagram";
+    uint8_t RawBigBuffer[1100] = {0};
     QUIC_BUFFER DatagramBuffer = { sizeof(RawBuffer), RawBuffer };
+    QUIC_BUFFER BigBuffer = { sizeof(RawBigBuffer), RawBigBuffer };
 
     SelectiveLossHelper LossHelper;
 
@@ -255,6 +257,22 @@ QuicTestDatagramSend(
                 TEST_TRUE(Client.GetDatagramsSuspectLost() > InitialLostCount);
                 CxPlatSleep(100);
 #endif
+                for (int i = 0; i < 20; i++) {
+                    TEST_QUIC_SUCCEEDED(
+                        MsQuic->DatagramSend(
+                            Client.GetConnection(),
+                            &BigBuffer,
+                            1,
+                            (i%2 == 0) ? QUIC_SEND_FLAG_CANCEL_ON_BLOCKED : QUIC_SEND_FLAG_NONE,
+                            nullptr));
+                }
+
+                Tries = 0;
+                while (Client.GetDatagramsCanceled() == 0 && ++Tries < 10) {
+                    CxPlatSleep(100);
+                }
+
+                TEST_NOT_EQUAL(0, Client.GetDatagramsCanceled());
 
                 Client.Shutdown(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, QUIC_TEST_NO_ERROR);
                 if (!Client.WaitForShutdownComplete()) {