diff --git a/IntegrationTests/tests_04_performance/Thresholds/5.10.json b/IntegrationTests/tests_04_performance/Thresholds/5.10.json index 67874b65af..9f75ed151e 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/5.10.json +++ b/IntegrationTests/tests_04_performance/Thresholds/5.10.json @@ -1,50 +1,50 @@ { "10000000_asyncsequenceproducer": 19, - "1000000_asyncwriter": 1000013, - "1000_addHandlers": 43000, - "1000_addHandlers_sync": 36000, - "1000_addRemoveHandlers_handlercontext": 8032, - "1000_addRemoveHandlers_handlername": 8032, - "1000_addRemoveHandlers_handlertype": 8032, - "1000_autoReadGetAndSet": 18000, + "1000000_asyncwriter": 1000050, + "1000_addHandlers": 43050, + "1000_addHandlers_sync": 36050, + "1000_addRemoveHandlers_handlercontext": 8050, + "1000_addRemoveHandlers_handlername": 8050, + "1000_addRemoveHandlers_handlertype": 8050, + "1000_autoReadGetAndSet": 18050, "1000_autoReadGetAndSet_sync": 0, - "1000_copying_bytebufferview_to_array": 1000, - "1000_copying_circularbuffer_to_array": 1000, - "1000_getHandlers": 8034, + "1000_copying_bytebufferview_to_array": 1050, + "1000_copying_circularbuffer_to_array": 1050, + "1000_getHandlers": 8050, "1000_getHandlers_sync": 34, - "1000_reqs_1_conn": 26363, - "1000_rst_connections": 145004, - "1000_tcpbootstraps": 3001, - "1000_tcpconnections": 152002, - "1000_udp_reqs": 6014, - "1000_udpbootstraps": 2000, - "1000_udpconnections": 75008, - "1_reqs_1000_conn": 389000, - "bytebuffer_lots_of_rw": 2005, + "1000_reqs_1_conn": 26400, + "1000_rst_connections": 145050, + "1000_tcpbootstraps": 3050, + "1000_tcpconnections": 152050, + "1000_udp_reqs": 6050, + "1000_udpbootstraps": 2050, + "1000_udpconnections": 75050, + "1_reqs_1000_conn": 389050, + "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, - "decode_1000_ws_frames": 2000, + "decode_1000_ws_frames": 2050, "encode_1000_ws_frames_holding_buffer": 3, - "encode_1000_ws_frames_holding_buffer_with_mask": 2003, + "encode_1000_ws_frames_holding_buffer_with_mask": 2050, "encode_1000_ws_frames_holding_buffer_with_space": 3, - "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2003, - "encode_1000_ws_frames_new_buffer": 3000, - "encode_1000_ws_frames_new_buffer_with_mask": 5000, - "encode_1000_ws_frames_new_buffer_with_space": 3000, - "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5000, + "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050, + "encode_1000_ws_frames_new_buffer": 3050, + "encode_1000_ws_frames_new_buffer_with_mask": 5050, + "encode_1000_ws_frames_new_buffer_with_space": 3050, + "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "future_erase_result": 4001, - "future_lots_of_callbacks": 53001, - "get_100000_headers_canonical_form": 700000, - "get_100000_headers_canonical_form_trimming_whitespace": 700000, - "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700000, - "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700000, + "future_erase_result": 4050, + "future_lots_of_callbacks": 53050, + "get_100000_headers_canonical_form": 700050, + "get_100000_headers_canonical_form_trimming_whitespace": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050, "modifying_1000_circular_buffer_elements": 0, - "modifying_byte_buffer_view": 6000, + "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 319, - "read_10000_chunks_from_file": 110009, - "schedule_10000_tasks": 40084, - "schedule_and_run_10000_tasks": 50010, + "read_10000_chunks_from_file": 110050, + "schedule_10000_tasks": 40100, + "schedule_and_run_10000_tasks": 50050, "scheduling_10000_executions": 89, - "udp_1000_reqs_1_conn": 6156, - "udp_1_reqs_1000_conn": 162000 + "udp_1000_reqs_1_conn": 6200, + "udp_1_reqs_1000_conn": 162050 } diff --git a/IntegrationTests/tests_04_performance/Thresholds/5.9.json b/IntegrationTests/tests_04_performance/Thresholds/5.9.json index b264515390..104b465358 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/5.9.json +++ b/IntegrationTests/tests_04_performance/Thresholds/5.9.json @@ -1,50 +1,50 @@ { "10000000_asyncsequenceproducer": 19, - "1000000_asyncwriter": 1000013, - "1000_addHandlers": 43000, - "1000_addHandlers_sync": 36000, - "1000_addRemoveHandlers_handlercontext": 8032, - "1000_addRemoveHandlers_handlername": 8032, - "1000_addRemoveHandlers_handlertype": 8032, - "1000_autoReadGetAndSet": 18000, + "1000000_asyncwriter": 1000050, + "1000_addHandlers": 43050, + "1000_addHandlers_sync": 36050, + "1000_addRemoveHandlers_handlercontext": 8050, + "1000_addRemoveHandlers_handlername": 8050, + "1000_addRemoveHandlers_handlertype": 8050, + "1000_autoReadGetAndSet": 18050, "1000_autoReadGetAndSet_sync": 0, - "1000_copying_bytebufferview_to_array": 1000, - "1000_copying_circularbuffer_to_array": 1000, - "1000_getHandlers": 8034, + "1000_copying_bytebufferview_to_array": 1050, + "1000_copying_circularbuffer_to_array": 1050, + "1000_getHandlers": 8050, "1000_getHandlers_sync": 34, - "1000_reqs_1_conn": 26372, - "1000_rst_connections": 147004, - "1000_tcpbootstraps": 4001, - "1000_tcpconnections": 154002, - "1000_udp_reqs": 6014, - "1000_udpbootstraps": 2000, - "1000_udpconnections": 75008, - "1_reqs_1000_conn": 398000, - "bytebuffer_lots_of_rw": 2005, + "1000_reqs_1_conn": 26400, + "1000_rst_connections": 147050, + "1000_tcpbootstraps": 4050, + "1000_tcpconnections": 154050, + "1000_udp_reqs": 6050, + "1000_udpbootstraps": 2050, + "1000_udpconnections": 75050, + "1_reqs_1000_conn": 398050, + "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, - "decode_1000_ws_frames": 2000, + "decode_1000_ws_frames": 2050, "encode_1000_ws_frames_holding_buffer": 3, - "encode_1000_ws_frames_holding_buffer_with_mask": 2003, + "encode_1000_ws_frames_holding_buffer_with_mask": 2050, "encode_1000_ws_frames_holding_buffer_with_space": 3, - "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2003, - "encode_1000_ws_frames_new_buffer": 3000, - "encode_1000_ws_frames_new_buffer_with_mask": 5000, - "encode_1000_ws_frames_new_buffer_with_space": 3000, - "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5000, + "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050, + "encode_1000_ws_frames_new_buffer": 3050, + "encode_1000_ws_frames_new_buffer_with_mask": 5050, + "encode_1000_ws_frames_new_buffer_with_space": 3050, + "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "future_erase_result": 4001, - "future_lots_of_callbacks": 53001, - "get_100000_headers_canonical_form": 700000, - "get_100000_headers_canonical_form_trimming_whitespace": 700000, - "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700000, - "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700000, + "future_erase_result": 4050, + "future_lots_of_callbacks": 53050, + "get_100000_headers_canonical_form": 700050, + "get_100000_headers_canonical_form_trimming_whitespace": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050, "modifying_1000_circular_buffer_elements": 0, - "modifying_byte_buffer_view": 6000, + "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 334, - "read_10000_chunks_from_file": 110009, - "schedule_10000_tasks": 40084, - "schedule_and_run_10000_tasks": 50010, + "read_10000_chunks_from_file": 110050, + "schedule_10000_tasks": 40100, + "schedule_and_run_10000_tasks": 50050, "scheduling_10000_executions": 89, - "udp_1000_reqs_1_conn": 6156, - "udp_1_reqs_1000_conn": 162000 + "udp_1000_reqs_1_conn": 6200, + "udp_1_reqs_1000_conn": 162050 } diff --git a/IntegrationTests/tests_04_performance/Thresholds/6.0.json b/IntegrationTests/tests_04_performance/Thresholds/6.0.json new file mode 100644 index 0000000000..9f75ed151e --- /dev/null +++ b/IntegrationTests/tests_04_performance/Thresholds/6.0.json @@ -0,0 +1,50 @@ +{ + "10000000_asyncsequenceproducer": 19, + "1000000_asyncwriter": 1000050, + "1000_addHandlers": 43050, + "1000_addHandlers_sync": 36050, + "1000_addRemoveHandlers_handlercontext": 8050, + "1000_addRemoveHandlers_handlername": 8050, + "1000_addRemoveHandlers_handlertype": 8050, + "1000_autoReadGetAndSet": 18050, + "1000_autoReadGetAndSet_sync": 0, + "1000_copying_bytebufferview_to_array": 1050, + "1000_copying_circularbuffer_to_array": 1050, + "1000_getHandlers": 8050, + "1000_getHandlers_sync": 34, + "1000_reqs_1_conn": 26400, + "1000_rst_connections": 145050, + "1000_tcpbootstraps": 3050, + "1000_tcpconnections": 152050, + "1000_udp_reqs": 6050, + "1000_udpbootstraps": 2050, + "1000_udpconnections": 75050, + "1_reqs_1000_conn": 389050, + "bytebuffer_lots_of_rw": 2050, + "creating_10000_headers": 0, + "decode_1000_ws_frames": 2050, + "encode_1000_ws_frames_holding_buffer": 3, + "encode_1000_ws_frames_holding_buffer_with_mask": 2050, + "encode_1000_ws_frames_holding_buffer_with_space": 3, + "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050, + "encode_1000_ws_frames_new_buffer": 3050, + "encode_1000_ws_frames_new_buffer_with_mask": 5050, + "encode_1000_ws_frames_new_buffer_with_space": 3050, + "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, + "execute_hop_10000_tasks": 0, + "future_erase_result": 4050, + "future_lots_of_callbacks": 53050, + "get_100000_headers_canonical_form": 700050, + "get_100000_headers_canonical_form_trimming_whitespace": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050, + "modifying_1000_circular_buffer_elements": 0, + "modifying_byte_buffer_view": 6050, + "ping_pong_1000_reqs_1_conn": 319, + "read_10000_chunks_from_file": 110050, + "schedule_10000_tasks": 40100, + "schedule_and_run_10000_tasks": 50050, + "scheduling_10000_executions": 89, + "udp_1000_reqs_1_conn": 6200, + "udp_1_reqs_1000_conn": 162050 +} diff --git a/IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json b/IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json index 67874b65af..9f75ed151e 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json +++ b/IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json @@ -1,50 +1,50 @@ { "10000000_asyncsequenceproducer": 19, - "1000000_asyncwriter": 1000013, - "1000_addHandlers": 43000, - "1000_addHandlers_sync": 36000, - "1000_addRemoveHandlers_handlercontext": 8032, - "1000_addRemoveHandlers_handlername": 8032, - "1000_addRemoveHandlers_handlertype": 8032, - "1000_autoReadGetAndSet": 18000, + "1000000_asyncwriter": 1000050, + "1000_addHandlers": 43050, + "1000_addHandlers_sync": 36050, + "1000_addRemoveHandlers_handlercontext": 8050, + "1000_addRemoveHandlers_handlername": 8050, + "1000_addRemoveHandlers_handlertype": 8050, + "1000_autoReadGetAndSet": 18050, "1000_autoReadGetAndSet_sync": 0, - "1000_copying_bytebufferview_to_array": 1000, - "1000_copying_circularbuffer_to_array": 1000, - "1000_getHandlers": 8034, + "1000_copying_bytebufferview_to_array": 1050, + "1000_copying_circularbuffer_to_array": 1050, + "1000_getHandlers": 8050, "1000_getHandlers_sync": 34, - "1000_reqs_1_conn": 26363, - "1000_rst_connections": 145004, - "1000_tcpbootstraps": 3001, - "1000_tcpconnections": 152002, - "1000_udp_reqs": 6014, - "1000_udpbootstraps": 2000, - "1000_udpconnections": 75008, - "1_reqs_1000_conn": 389000, - "bytebuffer_lots_of_rw": 2005, + "1000_reqs_1_conn": 26400, + "1000_rst_connections": 145050, + "1000_tcpbootstraps": 3050, + "1000_tcpconnections": 152050, + "1000_udp_reqs": 6050, + "1000_udpbootstraps": 2050, + "1000_udpconnections": 75050, + "1_reqs_1000_conn": 389050, + "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, - "decode_1000_ws_frames": 2000, + "decode_1000_ws_frames": 2050, "encode_1000_ws_frames_holding_buffer": 3, - "encode_1000_ws_frames_holding_buffer_with_mask": 2003, + "encode_1000_ws_frames_holding_buffer_with_mask": 2050, "encode_1000_ws_frames_holding_buffer_with_space": 3, - "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2003, - "encode_1000_ws_frames_new_buffer": 3000, - "encode_1000_ws_frames_new_buffer_with_mask": 5000, - "encode_1000_ws_frames_new_buffer_with_space": 3000, - "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5000, + "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050, + "encode_1000_ws_frames_new_buffer": 3050, + "encode_1000_ws_frames_new_buffer_with_mask": 5050, + "encode_1000_ws_frames_new_buffer_with_space": 3050, + "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "future_erase_result": 4001, - "future_lots_of_callbacks": 53001, - "get_100000_headers_canonical_form": 700000, - "get_100000_headers_canonical_form_trimming_whitespace": 700000, - "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700000, - "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700000, + "future_erase_result": 4050, + "future_lots_of_callbacks": 53050, + "get_100000_headers_canonical_form": 700050, + "get_100000_headers_canonical_form_trimming_whitespace": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050, "modifying_1000_circular_buffer_elements": 0, - "modifying_byte_buffer_view": 6000, + "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 319, - "read_10000_chunks_from_file": 110009, - "schedule_10000_tasks": 40084, - "schedule_and_run_10000_tasks": 50010, + "read_10000_chunks_from_file": 110050, + "schedule_10000_tasks": 40100, + "schedule_and_run_10000_tasks": 50050, "scheduling_10000_executions": 89, - "udp_1000_reqs_1_conn": 6156, - "udp_1_reqs_1000_conn": 162000 + "udp_1000_reqs_1_conn": 6200, + "udp_1_reqs_1000_conn": 162050 } diff --git a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json index 67874b65af..9f75ed151e 100644 --- a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json +++ b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json @@ -1,50 +1,50 @@ { "10000000_asyncsequenceproducer": 19, - "1000000_asyncwriter": 1000013, - "1000_addHandlers": 43000, - "1000_addHandlers_sync": 36000, - "1000_addRemoveHandlers_handlercontext": 8032, - "1000_addRemoveHandlers_handlername": 8032, - "1000_addRemoveHandlers_handlertype": 8032, - "1000_autoReadGetAndSet": 18000, + "1000000_asyncwriter": 1000050, + "1000_addHandlers": 43050, + "1000_addHandlers_sync": 36050, + "1000_addRemoveHandlers_handlercontext": 8050, + "1000_addRemoveHandlers_handlername": 8050, + "1000_addRemoveHandlers_handlertype": 8050, + "1000_autoReadGetAndSet": 18050, "1000_autoReadGetAndSet_sync": 0, - "1000_copying_bytebufferview_to_array": 1000, - "1000_copying_circularbuffer_to_array": 1000, - "1000_getHandlers": 8034, + "1000_copying_bytebufferview_to_array": 1050, + "1000_copying_circularbuffer_to_array": 1050, + "1000_getHandlers": 8050, "1000_getHandlers_sync": 34, - "1000_reqs_1_conn": 26363, - "1000_rst_connections": 145004, - "1000_tcpbootstraps": 3001, - "1000_tcpconnections": 152002, - "1000_udp_reqs": 6014, - "1000_udpbootstraps": 2000, - "1000_udpconnections": 75008, - "1_reqs_1000_conn": 389000, - "bytebuffer_lots_of_rw": 2005, + "1000_reqs_1_conn": 26400, + "1000_rst_connections": 145050, + "1000_tcpbootstraps": 3050, + "1000_tcpconnections": 152050, + "1000_udp_reqs": 6050, + "1000_udpbootstraps": 2050, + "1000_udpconnections": 75050, + "1_reqs_1000_conn": 389050, + "bytebuffer_lots_of_rw": 2050, "creating_10000_headers": 0, - "decode_1000_ws_frames": 2000, + "decode_1000_ws_frames": 2050, "encode_1000_ws_frames_holding_buffer": 3, - "encode_1000_ws_frames_holding_buffer_with_mask": 2003, + "encode_1000_ws_frames_holding_buffer_with_mask": 2050, "encode_1000_ws_frames_holding_buffer_with_space": 3, - "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2003, - "encode_1000_ws_frames_new_buffer": 3000, - "encode_1000_ws_frames_new_buffer_with_mask": 5000, - "encode_1000_ws_frames_new_buffer_with_space": 3000, - "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5000, + "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050, + "encode_1000_ws_frames_new_buffer": 3050, + "encode_1000_ws_frames_new_buffer_with_mask": 5050, + "encode_1000_ws_frames_new_buffer_with_space": 3050, + "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050, "execute_hop_10000_tasks": 0, - "future_erase_result": 4001, - "future_lots_of_callbacks": 53001, - "get_100000_headers_canonical_form": 700000, - "get_100000_headers_canonical_form_trimming_whitespace": 700000, - "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700000, - "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700000, + "future_erase_result": 4050, + "future_lots_of_callbacks": 53050, + "get_100000_headers_canonical_form": 700050, + "get_100000_headers_canonical_form_trimming_whitespace": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050, + "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050, "modifying_1000_circular_buffer_elements": 0, - "modifying_byte_buffer_view": 6000, + "modifying_byte_buffer_view": 6050, "ping_pong_1000_reqs_1_conn": 319, - "read_10000_chunks_from_file": 110009, - "schedule_10000_tasks": 40084, - "schedule_and_run_10000_tasks": 50010, + "read_10000_chunks_from_file": 110050, + "schedule_10000_tasks": 40100, + "schedule_and_run_10000_tasks": 50050, "scheduling_10000_executions": 89, - "udp_1000_reqs_1_conn": 6156, - "udp_1_reqs_1000_conn": 162000 + "udp_1000_reqs_1_conn": 6200, + "udp_1_reqs_1000_conn": 162050 } diff --git a/IntegrationTests/tests_04_performance/test_01_allocation_counts.sh b/IntegrationTests/tests_04_performance/test_01_allocation_counts.sh index 6b799ca3a8..6f233107c4 100755 --- a/IntegrationTests/tests_04_performance/test_01_allocation_counts.sh +++ b/IntegrationTests/tests_04_performance/test_01_allocation_counts.sh @@ -31,6 +31,20 @@ done "$here/test_01_resources/run-nio-alloc-counter-tests.sh" -t "$tmp" > "$tmp/output" +observed_allocations="{" +for test in "${all_tests[@]}"; do + while read -r test_case; do + test_case=${test_case#test_*} + total_allocations=$(grep "^test_$test_case.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') + observed_allocations="${observed_allocations} + \"${test_case}\": ${total_allocations}," + done < <(grep "^test_${test}[^\W]*.total_allocations:" "$tmp/output" | cut -d: -f1 | cut -d. -f1 | sort | uniq) +done +observed_allocations="${observed_allocations} +}" +info "observed allocations: +${observed_allocations}" + for test in "${all_tests[@]}"; do cat "$tmp/output" # helps debugging @@ -63,5 +77,5 @@ for test in "${all_tests[@]}"; do assert_less_than_or_equal "$total_allocations" "$max_allowed" assert_greater_than "$total_allocations" "$(( max_allowed - 1000))" fi - done < <(grep "^test_${test[^\W]}*.total_allocations:" "$tmp/output" | cut -d: -f1 | cut -d. -f1 | sort | uniq) + done < <(grep "^test_${test}[^\W]*.total_allocations:" "$tmp/output" | cut -d: -f1 | cut -d. -f1 | sort | uniq) done