Skip to content

Commit

Permalink
Speed up bulkUpdate (#60)
Browse files Browse the repository at this point in the history
Previously we were calling bulkUpdate on all participants
regardless of whether the fetched participants were actually
different from the ones stored in IndexedDB.

Since the participant data changes infrequently, we were
wasting a lot of CPU cycles on updating identical records.

We now filter out using deep equality only those participants
that are actually different and only bulkUpdate those.

For 10k participants, this leads to ~6x speedup.
  • Loading branch information
tomasr8 authored Jul 19, 2024
1 parent 10cb387 commit fe071e4
Show file tree
Hide file tree
Showing 5 changed files with 151 additions and 11 deletions.
26 changes: 22 additions & 4 deletions src/db/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import Dexie, {type EntityTable} from 'dexie';
import {useLiveQuery} from 'dexie-react-hooks';
import {FieldProps} from '../pages/participant/fields';
import {IndicoEvent, IndicoParticipant, IndicoRegform} from '../utils/client';
import {deepEqual} from '../utils/deep_equal';

export type IDBBoolean = 1 | 0; // IndexedDB doesn't support indexing booleans, so we use {1,0} instead

Expand Down Expand Up @@ -393,11 +394,28 @@ export async function updateParticipant(id: number, data: IndicoParticipant) {
}

export async function updateParticipants(ids: number[], participants: IndicoParticipant[]) {
const updates = participants.map(({id: indicoId, eventId, regformId, ...rest}, i) => ({
key: ids[i],
changes: {indicoId, ...rest},
const newParticipants = participants.map(({id: indicoId, eventId, regformId, ...rest}) => ({
indicoId,
...rest,
}));
await db.participants.bulkUpdate(updates);

return db.transaction('readwrite', db.participants, async () => {
const storedParticipants = await db.participants.bulkGet(ids);
const changes = [];
for (const [i, p] of storedParticipants.entries()) {
if (!p) {
continue;
}
const keysToCompare = new Set(Object.keys(newParticipants[i]));
const pChanges = Object.fromEntries(
Object.entries(p).filter(([key]) => keysToCompare.has(key))
);
if (!deepEqual(pChanges, newParticipants[i])) {
changes.push({key: ids[i], changes: newParticipants[i]});
}
}
await db.participants.bulkUpdate(changes);
});
}

function isFirefox() {
Expand Down
63 changes: 57 additions & 6 deletions src/pages/Events/sync.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -321,34 +321,67 @@ describe('test syncParticipants()', () => {
const participants = [
{
id: 10,
eventId: 50,
regformId: 60,
email: 'john.doe@cern.ch',
fullName: 'John Doe',
registrationDate: '2020-01-01',
registrationDate: '2023-11-06T14:29:26.485560+00:00',
state: 'complete',
price: 330,
currency: 'EUR',
formattedPrice: '€330.00',
isPaid: true,
paymentDate: '2023-11-23T13:20:26.703955+00:00',
checkedIn: true,
checkedInDt: '2020-01-02',
checkinSecret: '0000',
occupiedSlots: 3,
tags: [],
},
{
id: 30,
fullName: 'Jane Doe',
registrationDate: '2020-03-01',
eventId: 50,
regformId: 60,
email: 'jane.doe@cern.ch',
registrationDate: '2023-12-06T14:29:26.485560+00:00',
state: 'unpaid',
price: 120,
currency: 'EUR',
formattedPrice: '€120.00',
isPaid: false,
checkedIn: true,
checkedInDt: null,
checkinSecret: '1111',
occupiedSlots: 1,
tags: [],
},
];
getParticipants.mockResolvedValue({ok: true, data: participants});

const storedEvent = {id: 1, serverId: 1};
const storedRegform = {id: 3, eventId: 1};
const storedParticipants = [
{id: 1, indicoId: 10, regformId: 3, registrationData: []},
{id: 2, indicoId: 20, regformId: 3, registrationData: []},
{
id: 1,
indicoId: 10,
regformId: 3,
registrationData: [],
notes: '',
deleted: 0,
checkedInLoading: 0,
isPaidLoading: 0,
},
{
id: 2,
indicoId: 20,
regformId: 3,
registrationData: [],
notes: '',
deleted: 0,
checkedInLoading: 0,
isPaidLoading: 0,
},
];
await db.events.add(storedEvent);
await db.regforms.add(storedRegform);
Expand All @@ -362,29 +395,43 @@ describe('test syncParticipants()', () => {
id: 1,
indicoId: 10,
regformId: 3,
email: 'john.doe@cern.ch',
fullName: 'John Doe',
registrationDate: '2020-01-01',
registrationDate: '2023-11-06T14:29:26.485560+00:00',
registrationData: [],
state: 'complete',
checkedIn: true,
checkedInDt: '2020-01-02',
checkedInLoading: 0,
checkinSecret: '0000',
occupiedSlots: 3,
isPaid: true,
isPaidLoading: 0,
paymentDate: '2023-11-23T13:20:26.703955+00:00',
price: 330,
currency: 'EUR',
formattedPrice: '€330.00',
notes: '',
deleted: 0,
tags: [],
},
{
id: 2,
indicoId: 20,
regformId: 3,
registrationData: [],
checkedInLoading: 0,
isPaidLoading: 0,
notes: '',
deleted: 1,
},
{
id: 3,
indicoId: 30,
regformId: 3,
email: 'jane.doe@cern.ch',
fullName: 'Jane Doe',
registrationDate: '2020-03-01',
registrationDate: '2023-12-06T14:29:26.485560+00:00',
state: 'unpaid',
checkedIn: true,
checkedInDt: null,
Expand All @@ -393,8 +440,12 @@ describe('test syncParticipants()', () => {
occupiedSlots: 1,
isPaid: false,
isPaidLoading: 0,
price: 120,
currency: 'EUR',
formattedPrice: '€120.00',
notes: '',
deleted: 0,
tags: [],
},
]);
expect(errorModal).not.toHaveBeenCalled();
Expand Down
2 changes: 1 addition & 1 deletion src/pages/Events/sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ export async function syncParticipants(
const deleted = onlyExisting.map(r => ({key: r.id, changes: {deleted: 1 as IDBBoolean}}));
await db.participants.bulkUpdate(deleted);
// participants that we don't have locally, add them
const newData = onlyNew.map(({id, ...p}) => ({
const newData = onlyNew.map(({id, eventId, regformId, ...p}) => ({
...p,
indicoId: id,
regformId: regform.id,
Expand Down
25 changes: 25 additions & 0 deletions src/utils/deep_equal.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import {deepEqual} from './deep_equal';

test('test deepEqual()', () => {
expect(deepEqual(undefined, undefined)).toBe(true);
expect(deepEqual(null, null)).toBe(true);
expect(deepEqual(42, 42)).toBe(true);
expect(deepEqual('foo', 'foo')).toBe(true);

expect(deepEqual(42, null)).toBe(false);
expect(deepEqual(null, 42)).toBe(false);

expect(deepEqual([], [])).toBe(true);
expect(deepEqual([1, 2, 3], [1, 2, 3])).toBe(true);
expect(deepEqual([1, 2, 3], [1, 2, 4])).toBe(false);
expect(deepEqual([1, 2, 3], [1, 2])).toBe(false);
expect(deepEqual([1, 2], [1, 2, 3])).toBe(false);

expect(deepEqual({}, {})).toBe(true);
expect(deepEqual({a: 1, b: 2}, {a: 1, b: 2})).toBe(true);
expect(deepEqual({a: 1, b: 2}, {a: 1, b: 3})).toBe(false);
expect(deepEqual({a: 1, b: 2}, {a: 1})).toBe(false);
expect(deepEqual({a: 1}, {a: 1, b: 2})).toBe(false);

expect(deepEqual('foo', 'bar')).toBe(false);
});
46 changes: 46 additions & 0 deletions src/utils/deep_equal.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
export function deepEqual(a: any, b: any): boolean {
if (a === b) {
return true;
}

if (a === null || b === null) {
return false;
}

if (Array.isArray(a) && Array.isArray(b)) {
return deepEqualArray(a, b);
}

if (typeof a === 'object' && typeof b === 'object') {
return deepEqualObject(a, b);
}

return false;
}

function deepEqualArray(a: any[], b: any[]): boolean {
if (a.length !== b.length) {
return false;
}

for (let i = 0; i < a.length; i++) {
if (!deepEqual(a[i], b[i])) {
return false;
}
}
return true;
}

function deepEqualObject(a: {[key: string]: any}, b: {[key: string]: any}): boolean {
const keys = Object.keys(a);
if (keys.length !== Object.keys(b).length) {
return false;
}

for (const key of keys) {
if (!deepEqual(a[key], b[key])) {
return false;
}
}
return true;
}

0 comments on commit fe071e4

Please sign in to comment.