From 45f23ada2c6ec22da6be1559f46921d5588f06e3 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Wed, 5 Jun 2024 22:39:15 +0200 Subject: [PATCH] Speed up bulkUpdate Previously we were calling bulkUpdate on all participants regardless of whether the fetched participants were actually different from the ones stored in IndexedDB. Since the participant data changes infrequently, we were wasting a lot of CPU cycles on updating identical records. We now filter out using deep equality only those participants that are actually different and only bulkUpdate those. For 10k participants, this leads to ~6x speedup. --- src/db/db.ts | 26 ++++++++++++--- src/pages/Events/sync.test.js | 63 +++++++++++++++++++++++++++++++---- src/pages/Events/sync.ts | 2 +- src/utils/deep_equal.test.ts | 25 ++++++++++++++ src/utils/deep_equal.ts | 46 +++++++++++++++++++++++++ 5 files changed, 151 insertions(+), 11 deletions(-) create mode 100644 src/utils/deep_equal.test.ts create mode 100644 src/utils/deep_equal.ts diff --git a/src/db/db.ts b/src/db/db.ts index cda1e5c..725dfec 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -2,6 +2,7 @@ import Dexie, {type EntityTable} from 'dexie'; import {useLiveQuery} from 'dexie-react-hooks'; import {FieldProps} from '../pages/participant/fields'; import {IndicoEvent, IndicoParticipant, IndicoRegform} from '../utils/client'; +import {deepEqual} from '../utils/deep_equal'; export type IDBBoolean = 1 | 0; // IndexedDB doesn't support indexing booleans, so we use {1,0} instead @@ -393,11 +394,28 @@ export async function updateParticipant(id: number, data: IndicoParticipant) { } export async function updateParticipants(ids: number[], participants: IndicoParticipant[]) { - const updates = participants.map(({id: indicoId, eventId, regformId, ...rest}, i) => ({ - key: ids[i], - changes: {indicoId, ...rest}, + const newParticipants = participants.map(({id: indicoId, eventId, regformId, ...rest}) => ({ + indicoId, + ...rest, })); - await db.participants.bulkUpdate(updates); + + return db.transaction('readwrite', db.participants, async () => { + const storedParticipants = await db.participants.bulkGet(ids); + const changes = []; + for (const [i, p] of storedParticipants.entries()) { + if (!p) { + continue; + } + const keysToCompare = new Set(Object.keys(newParticipants[i])); + const pChanges = Object.fromEntries( + Object.entries(p).filter(([key]) => keysToCompare.has(key)) + ); + if (!deepEqual(pChanges, newParticipants[i])) { + changes.push({key: ids[i], changes: newParticipants[i]}); + } + } + await db.participants.bulkUpdate(changes); + }); } function isFirefox() { diff --git a/src/pages/Events/sync.test.js b/src/pages/Events/sync.test.js index a9f499f..93bd4bf 100644 --- a/src/pages/Events/sync.test.js +++ b/src/pages/Events/sync.test.js @@ -321,25 +321,40 @@ describe('test syncParticipants()', () => { const participants = [ { id: 10, + eventId: 50, + regformId: 60, + email: 'john.doe@cern.ch', fullName: 'John Doe', - registrationDate: '2020-01-01', + registrationDate: '2023-11-06T14:29:26.485560+00:00', state: 'complete', + price: 330, + currency: 'EUR', + formattedPrice: '€330.00', isPaid: true, + paymentDate: '2023-11-23T13:20:26.703955+00:00', checkedIn: true, checkedInDt: '2020-01-02', checkinSecret: '0000', occupiedSlots: 3, + tags: [], }, { id: 30, fullName: 'Jane Doe', - registrationDate: '2020-03-01', + eventId: 50, + regformId: 60, + email: 'jane.doe@cern.ch', + registrationDate: '2023-12-06T14:29:26.485560+00:00', state: 'unpaid', + price: 120, + currency: 'EUR', + formattedPrice: '€120.00', isPaid: false, checkedIn: true, checkedInDt: null, checkinSecret: '1111', occupiedSlots: 1, + tags: [], }, ]; getParticipants.mockResolvedValue({ok: true, data: participants}); @@ -347,8 +362,26 @@ describe('test syncParticipants()', () => { const storedEvent = {id: 1, serverId: 1}; const storedRegform = {id: 3, eventId: 1}; const storedParticipants = [ - {id: 1, indicoId: 10, regformId: 3, registrationData: []}, - {id: 2, indicoId: 20, regformId: 3, registrationData: []}, + { + id: 1, + indicoId: 10, + regformId: 3, + registrationData: [], + notes: '', + deleted: 0, + checkedInLoading: 0, + isPaidLoading: 0, + }, + { + id: 2, + indicoId: 20, + regformId: 3, + registrationData: [], + notes: '', + deleted: 0, + checkedInLoading: 0, + isPaidLoading: 0, + }, ]; await db.events.add(storedEvent); await db.regforms.add(storedRegform); @@ -362,29 +395,43 @@ describe('test syncParticipants()', () => { id: 1, indicoId: 10, regformId: 3, + email: 'john.doe@cern.ch', fullName: 'John Doe', - registrationDate: '2020-01-01', + registrationDate: '2023-11-06T14:29:26.485560+00:00', registrationData: [], state: 'complete', checkedIn: true, checkedInDt: '2020-01-02', + checkedInLoading: 0, checkinSecret: '0000', occupiedSlots: 3, isPaid: true, + isPaidLoading: 0, + paymentDate: '2023-11-23T13:20:26.703955+00:00', + price: 330, + currency: 'EUR', + formattedPrice: '€330.00', + notes: '', + deleted: 0, + tags: [], }, { id: 2, indicoId: 20, regformId: 3, registrationData: [], + checkedInLoading: 0, + isPaidLoading: 0, + notes: '', deleted: 1, }, { id: 3, indicoId: 30, regformId: 3, + email: 'jane.doe@cern.ch', fullName: 'Jane Doe', - registrationDate: '2020-03-01', + registrationDate: '2023-12-06T14:29:26.485560+00:00', state: 'unpaid', checkedIn: true, checkedInDt: null, @@ -393,8 +440,12 @@ describe('test syncParticipants()', () => { occupiedSlots: 1, isPaid: false, isPaidLoading: 0, + price: 120, + currency: 'EUR', + formattedPrice: '€120.00', notes: '', deleted: 0, + tags: [], }, ]); expect(errorModal).not.toHaveBeenCalled(); diff --git a/src/pages/Events/sync.ts b/src/pages/Events/sync.ts index 802c5cc..aaa62cc 100644 --- a/src/pages/Events/sync.ts +++ b/src/pages/Events/sync.ts @@ -143,7 +143,7 @@ export async function syncParticipants( const deleted = onlyExisting.map(r => ({key: r.id, changes: {deleted: 1 as IDBBoolean}})); await db.participants.bulkUpdate(deleted); // participants that we don't have locally, add them - const newData = onlyNew.map(({id, ...p}) => ({ + const newData = onlyNew.map(({id, eventId, regformId, ...p}) => ({ ...p, indicoId: id, regformId: regform.id, diff --git a/src/utils/deep_equal.test.ts b/src/utils/deep_equal.test.ts new file mode 100644 index 0000000..35925f9 --- /dev/null +++ b/src/utils/deep_equal.test.ts @@ -0,0 +1,25 @@ +import {deepEqual} from './deep_equal'; + +test('test deepEqual()', () => { + expect(deepEqual(undefined, undefined)).toBe(true); + expect(deepEqual(null, null)).toBe(true); + expect(deepEqual(42, 42)).toBe(true); + expect(deepEqual('foo', 'foo')).toBe(true); + + expect(deepEqual(42, null)).toBe(false); + expect(deepEqual(null, 42)).toBe(false); + + expect(deepEqual([], [])).toBe(true); + expect(deepEqual([1, 2, 3], [1, 2, 3])).toBe(true); + expect(deepEqual([1, 2, 3], [1, 2, 4])).toBe(false); + expect(deepEqual([1, 2, 3], [1, 2])).toBe(false); + expect(deepEqual([1, 2], [1, 2, 3])).toBe(false); + + expect(deepEqual({}, {})).toBe(true); + expect(deepEqual({a: 1, b: 2}, {a: 1, b: 2})).toBe(true); + expect(deepEqual({a: 1, b: 2}, {a: 1, b: 3})).toBe(false); + expect(deepEqual({a: 1, b: 2}, {a: 1})).toBe(false); + expect(deepEqual({a: 1}, {a: 1, b: 2})).toBe(false); + + expect(deepEqual('foo', 'bar')).toBe(false); +}); diff --git a/src/utils/deep_equal.ts b/src/utils/deep_equal.ts new file mode 100644 index 0000000..5f3e4fa --- /dev/null +++ b/src/utils/deep_equal.ts @@ -0,0 +1,46 @@ +export function deepEqual(a: any, b: any): boolean { + if (a === b) { + return true; + } + + if (a === null || b === null) { + return false; + } + + if (Array.isArray(a) && Array.isArray(b)) { + return deepEqualArray(a, b); + } + + if (typeof a === 'object' && typeof b === 'object') { + return deepEqualObject(a, b); + } + + return false; +} + +function deepEqualArray(a: any[], b: any[]): boolean { + if (a.length !== b.length) { + return false; + } + + for (let i = 0; i < a.length; i++) { + if (!deepEqual(a[i], b[i])) { + return false; + } + } + return true; +} + +function deepEqualObject(a: {[key: string]: any}, b: {[key: string]: any}): boolean { + const keys = Object.keys(a); + if (keys.length !== Object.keys(b).length) { + return false; + } + + for (const key of keys) { + if (!deepEqual(a[key], b[key])) { + return false; + } + } + return true; +}