Skip to content

Commit

Permalink
Light updates test: Ensure subcriptions have been applied before call…
Browse files Browse the repository at this point in the history
…ing reducers (#1979)
  • Loading branch information
Centril authored Nov 12, 2024
1 parent 5bf5b2b commit 57fb9ad
Showing 1 changed file with 16 additions and 11 deletions.
27 changes: 16 additions & 11 deletions crates/sdk/tests/test-client/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1616,23 +1616,21 @@ fn exec_caller_alice_receives_reducer_callback_but_not_bob() {
.ok_or_else(|| anyhow::anyhow!("wrong value received: `{val}`, expected: `{eq}`"))
}

// Have two actors, Alice (0) and Bob (1), connect to the module.
let connect_counter = TestCounter::new();
let actors = ["alice", "bob"];
let conns = actors.map(|who| connect_with_then(&connect_counter, who, |b| b.with_light_mode(true), |_| {}));
// Ensure both have finished connecting so that there isn't a race condition
// between Alice executing the reducer and Bob being connected.
connect_counter.wait_for_all();

let counter = TestCounter::new();
let pre_ins_counter = TestCounter::new();

// Have two actors, Alice (0) and Bob (1), connect to the module.
// For each actor, subscribe to the `OneU8` table.
// The choice of table is a fairly random one: just one of the simpler tables.
for (who, conn) in actors.into_iter().zip(conns.iter()) {
let sub_applied = counter.add_test(format!("sub_applied_{who}"));
let conns = ["alice", "bob"].map(|who| {
let conn = connect_with_then(&pre_ins_counter, who, |b| b.with_light_mode(true), |_| {});
let sub_applied = pre_ins_counter.add_test(format!("sub_applied_{who}"));

let counter2 = counter.clone();
conn.subscription_builder()
.on_applied(move |ctx| {
sub_applied(Ok(()));

// Test that we are notified when a row is inserted.
let db = ctx.db();
let mut one_u8_inserted = Some(counter2.add_test(format!("one_u8_inserted_{who}")));
Expand All @@ -1653,7 +1651,14 @@ fn exec_caller_alice_receives_reducer_callback_but_not_bob() {
})
.on_error(|_| panic!("Subscription error"))
.subscribe(["SELECT * FROM one_u8", "SELECT * FROM one_u16"]);
}
conn
});

// Ensure both have finished connecting
// and finished subscribing so that there isn't a race condition
// between Alice executing the reducer and Bob being connected
// or Alice executing the reducer and either having subscriptions applied.
pre_ins_counter.wait_for_all();

// Alice executes a reducer.
// This should cause a row callback to be received by Alice and Bob.
Expand Down

2 comments on commit 57fb9ad

@github-actions
Copy link

@github-actions github-actions bot commented on 57fb9ad Nov 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Criterion benchmark results

Criterion benchmark report

YOU SHOULD PROBABLY IGNORE THESE RESULTS.

Criterion is a wall time based benchmarking system that is extremely noisy when run on CI. We collect these results for longitudinal analysis, but they are not reliable for comparing individual PRs.

Go look at the callgrind report instead.

empty

db on disk new latency old latency new throughput old throughput
sqlite 💿 421.3±1.80ns 411.6±1.79ns - -
sqlite 🧠 413.6±1.18ns 402.8±2.00ns - -
stdb_raw 💿 779.3±3.76ns 773.3±0.72ns - -
stdb_raw 🧠 777.7±1.16ns 774.5±1.52ns - -

insert_1

db on disk schema indices preload new latency old latency new throughput old throughput

insert_bulk

db on disk schema indices preload count new latency old latency new throughput old throughput
sqlite 💿 u32_u64_str btree_each_column 2048 256 584.6±0.45µs 580.2±0.82µs 1710 tx/sec 1723 tx/sec
sqlite 💿 u32_u64_str unique_0 2048 256 150.0±0.52µs 149.7±0.47µs 6.5 Ktx/sec 6.5 Ktx/sec
sqlite 💿 u32_u64_u64 btree_each_column 2048 256 480.0±42.66µs 481.3±52.76µs 2.0 Ktx/sec 2.0 Ktx/sec
sqlite 💿 u32_u64_u64 unique_0 2048 256 136.6±1.01µs 138.4±0.80µs 7.1 Ktx/sec 7.1 Ktx/sec
sqlite 🧠 u32_u64_str btree_each_column 2048 256 447.6±3.58µs 445.1±0.98µs 2.2 Ktx/sec 2.2 Ktx/sec
sqlite 🧠 u32_u64_str unique_0 2048 256 126.9±0.36µs 121.4±0.74µs 7.7 Ktx/sec 8.0 Ktx/sec
sqlite 🧠 u32_u64_u64 btree_each_column 2048 256 368.0±0.24µs 367.4±0.67µs 2.7 Ktx/sec 2.7 Ktx/sec
sqlite 🧠 u32_u64_u64 unique_0 2048 256 104.6±0.54µs 104.3±0.41µs 9.3 Ktx/sec 9.4 Ktx/sec
stdb_raw 💿 u32_u64_str btree_each_column 2048 256 601.7±20.50µs 438.9±62.12µs 1661 tx/sec 2.2 Ktx/sec
stdb_raw 💿 u32_u64_str unique_0 2048 256 473.7±19.44µs 432.7±51.58µs 2.1 Ktx/sec 2.3 Ktx/sec
stdb_raw 💿 u32_u64_u64 btree_each_column 2048 256 376.0±9.37µs 370.2±20.08µs 2.6 Ktx/sec 2.6 Ktx/sec
stdb_raw 💿 u32_u64_u64 unique_0 2048 256 343.8±8.88µs 347.1±7.09µs 2.8 Ktx/sec 2.8 Ktx/sec
stdb_raw 🧠 u32_u64_str btree_each_column 2048 256 297.9±0.21µs 298.9±0.64µs 3.3 Ktx/sec 3.3 Ktx/sec
stdb_raw 🧠 u32_u64_str unique_0 2048 256 228.8±0.41µs 230.6±0.28µs 4.3 Ktx/sec 4.2 Ktx/sec
stdb_raw 🧠 u32_u64_u64 btree_each_column 2048 256 239.1±0.20µs 241.9±0.38µs 4.1 Ktx/sec 4.0 Ktx/sec
stdb_raw 🧠 u32_u64_u64 unique_0 2048 256 207.0±0.25µs 212.5±0.13µs 4.7 Ktx/sec 4.6 Ktx/sec

iterate

db on disk schema indices new latency old latency new throughput old throughput
sqlite 💿 u32_u64_str unique_0 23.4±0.11µs 23.8±0.42µs 41.8 Ktx/sec 41.0 Ktx/sec
sqlite 💿 u32_u64_u64 unique_0 22.1±0.06µs 22.0±0.07µs 44.2 Ktx/sec 44.5 Ktx/sec
sqlite 🧠 u32_u64_str unique_0 20.9±0.08µs 20.8±0.23µs 46.6 Ktx/sec 47.0 Ktx/sec
sqlite 🧠 u32_u64_u64 unique_0 19.5±0.08µs 19.4±0.07µs 50.2 Ktx/sec 50.2 Ktx/sec
stdb_raw 💿 u32_u64_str unique_0 4.0±0.00µs 4.9±0.00µs 245.9 Ktx/sec 199.6 Ktx/sec
stdb_raw 💿 u32_u64_u64 unique_0 3.9±0.00µs 4.8±0.00µs 251.8 Ktx/sec 204.1 Ktx/sec
stdb_raw 🧠 u32_u64_str unique_0 4.0±0.00µs 4.9±0.00µs 246.0 Ktx/sec 200.1 Ktx/sec
stdb_raw 🧠 u32_u64_u64 unique_0 3.9±0.00µs 4.8±0.00µs 251.8 Ktx/sec 204.0 Ktx/sec

find_unique

db on disk key type preload new latency old latency new throughput old throughput

filter

db on disk key type index strategy load count new latency old latency new throughput old throughput
sqlite 💿 string index 2048 256 70.4±0.35µs 70.9±0.53µs 13.9 Ktx/sec 13.8 Ktx/sec
sqlite 💿 u64 index 2048 256 66.9±0.12µs 66.6±0.11µs 14.6 Ktx/sec 14.7 Ktx/sec
sqlite 🧠 string index 2048 256 66.0±0.16µs 65.5±0.16µs 14.8 Ktx/sec 14.9 Ktx/sec
sqlite 🧠 u64 index 2048 256 60.0±0.28µs 60.1±0.10µs 16.3 Ktx/sec 16.3 Ktx/sec
stdb_raw 💿 string index 2048 256 5.0±0.00µs 4.9±0.00µs 195.7 Ktx/sec 198.6 Ktx/sec
stdb_raw 💿 u64 index 2048 256 4.9±0.00µs 4.8±0.00µs 198.8 Ktx/sec 201.5 Ktx/sec
stdb_raw 🧠 string index 2048 256 5.0±0.00µs 4.9±0.00µs 195.5 Ktx/sec 198.5 Ktx/sec
stdb_raw 🧠 u64 index 2048 256 4.9±0.00µs 4.8±0.00µs 198.7 Ktx/sec 201.7 Ktx/sec

serialize

schema format count new latency old latency new throughput old throughput
u32_u64_str bflatn_to_bsatn_fast_path 100 3.9±0.01µs 3.9±0.01µs 24.7 Mtx/sec 24.7 Mtx/sec
u32_u64_str bflatn_to_bsatn_slow_path 100 3.5±0.01µs 3.5±0.01µs 27.0 Mtx/sec 26.9 Mtx/sec
u32_u64_str bsatn 100 15.6±0.17ns 15.5±0.15ns 6.0 Gtx/sec 6.0 Gtx/sec
u32_u64_str bsatn 100 2.4±0.00µs 2.4±0.00µs 40.4 Mtx/sec 40.1 Mtx/sec
u32_u64_str json 100 5.0±0.03µs 4.8±0.06µs 19.2 Mtx/sec 19.9 Mtx/sec
u32_u64_str json 100 9.6±0.02µs 9.5±0.01µs 10.0 Mtx/sec 10.0 Mtx/sec
u32_u64_str product_value 100 1019.1±1.23ns 1017.4±0.94ns 93.6 Mtx/sec 93.7 Mtx/sec
u32_u64_u64 bflatn_to_bsatn_fast_path 100 1010.7±8.08ns 1005.7±1.36ns 94.4 Mtx/sec 94.8 Mtx/sec
u32_u64_u64 bflatn_to_bsatn_slow_path 100 2.8±0.00µs 2.8±0.00µs 34.5 Mtx/sec 34.4 Mtx/sec
u32_u64_u64 bsatn 100 14.8±0.01ns 14.8±0.01ns 6.3 Gtx/sec 6.3 Gtx/sec
u32_u64_u64 bsatn 100 1826.3±19.44ns 1806.1±23.51ns 52.2 Mtx/sec 52.8 Mtx/sec
u32_u64_u64 json 100 3.4±0.04µs 3.5±0.03µs 27.8 Mtx/sec 27.6 Mtx/sec
u32_u64_u64 json 100 5.9±0.08µs 5.7±0.07µs 16.3 Mtx/sec 16.7 Mtx/sec
u32_u64_u64 product_value 100 1015.7±0.71ns 1017.0±1.03ns 93.9 Mtx/sec 93.8 Mtx/sec
u64_u64_u32 bflatn_to_bsatn_fast_path 100 753.0±0.32ns 761.6±12.62ns 126.6 Mtx/sec 125.2 Mtx/sec
u64_u64_u32 bflatn_to_bsatn_slow_path 100 2.8±0.00µs 2.8±0.05µs 34.3 Mtx/sec 34.2 Mtx/sec
u64_u64_u32 bsatn 100 1715.6±19.68ns 1706.1±19.75ns 55.6 Mtx/sec 55.9 Mtx/sec
u64_u64_u32 bsatn 100 923.9±3.27ns 926.3±5.16ns 103.2 Mtx/sec 103.0 Mtx/sec
u64_u64_u32 json 100 3.3±0.17µs 3.4±0.03µs 29.0 Mtx/sec 27.9 Mtx/sec
u64_u64_u32 json 100 5.8±0.01µs 5.9±0.06µs 16.4 Mtx/sec 16.2 Mtx/sec
u64_u64_u32 product_value 100 1014.6±0.68ns 1016.0±0.81ns 94.0 Mtx/sec 93.9 Mtx/sec

stdb_module_large_arguments

arg size new latency old latency new throughput old throughput
64KiB 102.2±10.40µs 102.4±6.66µs - -

stdb_module_print_bulk

line count new latency old latency new throughput old throughput
1 52.4±6.81µs 52.3±5.04µs - -
100 598.3±12.12µs 600.4±7.12µs - -
1000 4.7±0.80ms 3.7±0.61ms - -

remaining

name new latency old latency new throughput old throughput
special/db_game/circles/load=10 51.3±2.38ms 35.7±0.41ms - -
special/db_game/circles/load=100 36.1±1.22ms 43.1±6.39ms - -
special/db_game/ia_loop/load=500 148.1±0.81ms 149.2±1.78ms - -
special/db_game/ia_loop/load=5000 5.3±0.05s 5.3±0.03s - -
sqlite/💿/update_bulk/u32_u64_str/unique_0/load=2048/count=256 54.3±0.20µs 54.2±0.24µs 18.0 Ktx/sec 18.0 Ktx/sec
sqlite/💿/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 46.3±0.09µs 47.0±0.17µs 21.1 Ktx/sec 20.8 Ktx/sec
sqlite/🧠/update_bulk/u32_u64_str/unique_0/load=2048/count=256 39.6±0.13µs 39.6±0.10µs 24.7 Ktx/sec 24.7 Ktx/sec
sqlite/🧠/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 34.7±0.11µs 35.3±0.08µs 28.1 Ktx/sec 27.7 Ktx/sec
stdb_module/💿/update_bulk/u32_u64_str/unique_0/load=2048/count=256 1242.6±13.61µs 1232.9±30.09µs 804 tx/sec 811 tx/sec
stdb_module/💿/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 998.1±5.79µs 954.0±11.80µs 1001 tx/sec 1048 tx/sec
stdb_raw/💿/update_bulk/u32_u64_str/unique_0/load=2048/count=256 615.3±41.19µs 614.2±44.02µs 1625 tx/sec 1628 tx/sec
stdb_raw/💿/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 480.8±7.76µs 444.8±27.12µs 2.0 Ktx/sec 2.2 Ktx/sec
stdb_raw/🧠/update_bulk/u32_u64_str/unique_0/load=2048/count=256 363.1±0.20µs 369.1±0.49µs 2.7 Ktx/sec 2.6 Ktx/sec
stdb_raw/🧠/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 323.5±0.46µs 330.4±0.41µs 3.0 Ktx/sec 3.0 Ktx/sec

@github-actions
Copy link

@github-actions github-actions bot commented on 57fb9ad Nov 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Callgrind benchmark results

Callgrind Benchmark Report

These benchmarks were run using callgrind,
an instruction-level profiler. They allow comparisons between sqlite (sqlite), SpacetimeDB running through a module (stdb_module), and the underlying SpacetimeDB data storage engine (stdb_raw). Callgrind emulates a CPU to collect the below estimates.

Measurement changes larger than five percent are in bold.

In-memory benchmarks

callgrind: empty transaction

db total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw 6397 6397 0.00% 6527 6527 0.00%
sqlite 5589 5589 0.00% 6117 6117 0.00%

callgrind: filter

db schema indices count preload _column data_type total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str no_index 64 128 1 u64 76591 76591 0.00% 77031 77039 -0.01%
stdb_raw u32_u64_str no_index 64 128 2 string 120178 119089 0.91% 120932 119787 0.96%
stdb_raw u32_u64_str btree_each_column 64 128 2 string 25081 25083 -0.01% 25735 25765 -0.12%
stdb_raw u32_u64_str btree_each_column 64 128 1 u64 24049 24049 0.00% 24609 24601 0.03%
sqlite u32_u64_str no_index 64 128 2 string 144695 144695 0.00% 146181 146185 -0.00%
sqlite u32_u64_str no_index 64 128 1 u64 124044 124044 0.00% 125272 125276 -0.00%
sqlite u32_u64_str btree_each_column 64 128 1 u64 131361 131361 0.00% 132857 132853 0.00%
sqlite u32_u64_str btree_each_column 64 128 2 string 134494 134494 0.00% 136176 136184 -0.01%

callgrind: insert bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 64 128 878993 878415 0.07% 934437 933649 0.08%
stdb_raw u32_u64_str btree_each_column 64 128 1031446 1027365 0.40% 1098746 1063357 3.33%
sqlite u32_u64_str unique_0 64 128 398320 398320 0.00% 414994 414998 -0.00%
sqlite u32_u64_str btree_each_column 64 128 983637 983637 0.00% 1021691 1021695 -0.00%

callgrind: iterate

db schema indices count total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 153724 153724 0.00% 153866 153866 0.00%
stdb_raw u32_u64_str unique_0 64 16749 16749 0.00% 16871 16875 -0.02%
sqlite u32_u64_str unique_0 1024 1067255 1067255 0.00% 1070691 1070691 0.00%
sqlite u32_u64_str unique_0 64 76201 76201 0.00% 77335 77335 0.00%

callgrind: serialize_product_value

count format total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
64 json 47528 47528 0.00% 50180 50180 0.00%
64 bsatn 25509 25509 0.00% 27719 27719 0.00%
16 bsatn 8200 8200 0.00% 9526 9526 0.00%
16 json 12188 12188 0.00% 14092 14092 0.00%

callgrind: update bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 1024 20120482 20528126 -1.99% 20703592 21178694 -2.24%
stdb_raw u32_u64_str unique_0 64 128 1287068 1287498 -0.03% 1329642 1329664 -0.00%
sqlite u32_u64_str unique_0 1024 1024 1802182 1802182 0.00% 1811474 1811474 0.00%
sqlite u32_u64_str unique_0 64 128 128528 128528 0.00% 131416 131416 0.00%
On-disk benchmarks

callgrind: empty transaction

db total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw 6402 6402 0.00% 6536 6536 0.00%
sqlite 5621 5621 0.00% 6175 6175 0.00%

callgrind: filter

db schema indices count preload _column data_type total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str no_index 64 128 1 u64 76596 76596 0.00% 77012 77020 -0.01%
stdb_raw u32_u64_str no_index 64 128 2 string 119094 120183 -0.91% 119764 120981 -1.01%
stdb_raw u32_u64_str btree_each_column 64 128 2 string 25086 25088 -0.01% 25728 25734 -0.02%
stdb_raw u32_u64_str btree_each_column 64 128 1 u64 24054 24054 0.00% 24582 24586 -0.02%
sqlite u32_u64_str no_index 64 128 1 u64 125971 125965 0.00% 127555 127553 0.00%
sqlite u32_u64_str no_index 64 128 2 string 146616 146622 -0.00% 148458 148460 -0.00%
sqlite u32_u64_str btree_each_column 64 128 2 string 136616 136616 0.00% 138824 138824 0.00%
sqlite u32_u64_str btree_each_column 64 128 1 u64 133457 133457 0.00% 135343 135343 0.00%

callgrind: insert bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 64 128 828159 827694 0.06% 882163 850878 3.68%
stdb_raw u32_u64_str btree_each_column 64 128 978051 980393 -0.24% 1042435 1045247 -0.27%
sqlite u32_u64_str unique_0 64 128 415857 415857 0.00% 431843 431843 0.00%
sqlite u32_u64_str btree_each_column 64 128 1021898 1021904 -0.00% 1059298 1059288 0.00%

callgrind: iterate

db schema indices count total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 153729 153729 0.00% 153851 153847 0.00%
stdb_raw u32_u64_str unique_0 64 16754 16754 0.00% 16872 16872 0.00%
sqlite u32_u64_str unique_0 1024 1070323 1070323 0.00% 1074105 1074105 0.00%
sqlite u32_u64_str unique_0 64 77973 77973 0.00% 79263 79263 0.00%

callgrind: serialize_product_value

count format total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
64 json 47528 47528 0.00% 50180 50180 0.00%
64 bsatn 25509 25509 0.00% 27719 27719 0.00%
16 bsatn 8200 8200 0.00% 9526 9526 0.00%
16 json 12188 12188 0.00% 14092 14092 0.00%

callgrind: update bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 1024 19035674 19035086 0.00% 19691388 19681346 0.05%
stdb_raw u32_u64_str unique_0 64 128 1239755 1239434 0.03% 1312385 1312090 0.02%
sqlite u32_u64_str unique_0 1024 1024 1809743 1809743 0.00% 1818423 1818423 0.00%
sqlite u32_u64_str unique_0 64 128 132654 132654 0.00% 135606 135606 0.00%

Please sign in to comment.