From adf49789429ac5b77aafbbd02b47984a9f9ba397 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Tue, 5 Nov 2024 16:40:46 +0100 Subject: [PATCH] refactor & fix key iteration --- packages/chopsticks/package.json | 2 +- .../src/utils/fetch-storages.test.ts | 2 +- .../chopsticks/src/utils/fetch-storages.ts | 2 +- packages/core/package.json | 2 +- .../src/blockchain/get-keys-paged.test.ts | 189 +++++++++++++++++- packages/core/src/blockchain/storage-layer.ts | 168 ++++------------ packages/db/package.json | 2 +- packages/testing/package.json | 2 +- packages/utils/package.json | 2 +- 9 files changed, 223 insertions(+), 148 deletions(-) diff --git a/packages/chopsticks/package.json b/packages/chopsticks/package.json index 17083c32..98f18bc9 100644 --- a/packages/chopsticks/package.json +++ b/packages/chopsticks/package.json @@ -1,6 +1,6 @@ { "name": "@acala-network/chopsticks", - "version": "1.0.0", + "version": "1.0.1-1", "author": "Acala Developers ", "license": "Apache-2.0", "bin": "./chopsticks.cjs", diff --git a/packages/chopsticks/src/utils/fetch-storages.test.ts b/packages/chopsticks/src/utils/fetch-storages.test.ts index c31ed470..fc5a85a7 100644 --- a/packages/chopsticks/src/utils/fetch-storages.test.ts +++ b/packages/chopsticks/src/utils/fetch-storages.test.ts @@ -18,7 +18,7 @@ describe('fetch-storages', () => { beforeAll(async () => { provider = new WsProvider(endpoint, 30_000) - api = new ApiPromise({ provider }) + api = new ApiPromise({ provider, noInitWarn: true }) await api.isReady }) diff --git a/packages/chopsticks/src/utils/fetch-storages.ts b/packages/chopsticks/src/utils/fetch-storages.ts index bc941fce..7aeccd64 100644 --- a/packages/chopsticks/src/utils/fetch-storages.ts +++ b/packages/chopsticks/src/utils/fetch-storages.ts @@ -127,7 +127,7 @@ export const fetchStorages = async ({ block, endpoint, dbPath, config }: FetchSt if (!endpoint) throw new Error('endpoint is required') const provider = new WsProvider(endpoint, 3_000) - const apiPromise = new ApiPromise({ provider }) + const apiPromise = new ApiPromise({ provider, noInitWarn: true }) await apiPromise.isReady let blockHash: string diff --git a/packages/core/package.json b/packages/core/package.json index 8c8fea81..25b2092c 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@acala-network/chopsticks-core", - "version": "1.0.0", + "version": "1.0.1-1", "author": "Acala Developers ", "license": "Apache-2.0", "type": "module", diff --git a/packages/core/src/blockchain/get-keys-paged.test.ts b/packages/core/src/blockchain/get-keys-paged.test.ts index 692c64c8..67f7578f 100644 --- a/packages/core/src/blockchain/get-keys-paged.test.ts +++ b/packages/core/src/blockchain/get-keys-paged.test.ts @@ -1,12 +1,12 @@ import { Api } from '../api.js' -import { RemoteStorageLayer, StorageLayer, StorageValueKind } from './storage-layer.js' +import { RemoteStorageLayer, StorageLayer, StorageValue, StorageValueKind } from './storage-layer.js' import { describe, expect, it, vi } from 'vitest' import _ from 'lodash' describe('getKeysPaged', () => { const hash = '0x1111111111111111111111111111111111111111111111111111111111111111' - const allKeys = [ + const remoteKeys = [ '0x0000000000000000000000000000000000000000000000000000000000000000_00', '0x0000000000000000000000000000000000000000000000000000000000000000_03', '0x0000000000000000000000000000000000000000000000000000000000000000_04', @@ -22,10 +22,13 @@ describe('getKeysPaged', () => { ] Api.prototype['getKeysPaged'] = vi.fn(async (prefix, pageSize, startKey) => { - const withPrefix = allKeys.filter((k) => k.startsWith(prefix) && k > startKey) + const withPrefix = remoteKeys.filter((k) => k.startsWith(prefix) && k > startKey) const result = withPrefix.slice(0, pageSize) return result }) + Api.prototype['getStorage'] = vi.fn(async (_key, _at) => { + return '0x1' as any + }) const mockApi = new Api(undefined!) const remoteLayer = new RemoteStorageLayer(mockApi, hash, undefined) @@ -90,6 +93,17 @@ describe('getKeysPaged', () => { }) it('remote layer works', async () => { + expect( + await remoteLayer.getKeysPaged( + '0x0000000000000000000000000000000000000000000000000000000000000000', + 10, + '0x0000000000000000000000000000000000000000000000000000000000000000', + ), + ).toEqual([ + '0x0000000000000000000000000000000000000000000000000000000000000000_00', + '0x0000000000000000000000000000000000000000000000000000000000000000_03', + '0x0000000000000000000000000000000000000000000000000000000000000000_04', + ]) expect( await remoteLayer.getKeysPaged( '0x1111111111111111111111111111111111111111111111111111111111111111', @@ -140,10 +154,7 @@ describe('getKeysPaged', () => { it('updated values', async () => { const layer2 = new StorageLayer(storageLayer) - layer2.setAll([ - ['0x1111111111111111111111111111111111111111111111111111111111111111_00', '0x00'], - ['0x1111111111111111111111111111111111111111111111111111111111111111_04', '0x04'], - ]) + layer2.setAll([['0x1111111111111111111111111111111111111111111111111111111111111111_04', '0x04']]) expect( await layer2.getKeysPaged( '0x1111111111111111111111111111111111111111111111111111111111111111', @@ -156,6 +167,20 @@ describe('getKeysPaged', () => { '0x1111111111111111111111111111111111111111111111111111111111111111_07', ]) + expect( + await layer2.getKeysPaged( + '0x1111111111111111111111111111111111111111111111111111111111111111', + 4, + '0x1111111111111111111111111111111111111111111111111111111111111111', + ), + ).toEqual([ + '0x1111111111111111111111111111111111111111111111111111111111111111_01', + '0x1111111111111111111111111111111111111111111111111111111111111111_02', + '0x1111111111111111111111111111111111111111111111111111111111111111_03', + '0x1111111111111111111111111111111111111111111111111111111111111111_04', + ]) + + layer2.setAll([['0x1111111111111111111111111111111111111111111111111111111111111111_00', '0x00']]) expect( await layer2.getKeysPaged( '0x1111111111111111111111111111111111111111111111111111111111111111', @@ -303,5 +328,155 @@ describe('getKeysPaged', () => { '0xcd710b30bd2eab0352ddcc26417aa19463c716fb8fff3de61a883bb76adb34a2', ), ).toEqual([]) + + layer3.setAll([['0xcd710b30bd2eab0352ddcc26417aa19463c716fb8fff3de61a883bb76adb34a2_01', '0x01']]) + expect( + await layer3.getKeysPaged( + '0xcd710b30bd2eab0352ddcc26417aa19463c716fb8fff3de61a883bb76adb34a2', + 1, + '0xcd710b30bd2eab0352ddcc26417aa19463c716fb8fff3de61a883bb76adb34a2', + ), + ).toEqual(['0xcd710b30bd2eab0352ddcc26417aa19463c716fb8fff3de61a883bb76adb34a2_01']) + }) + + it('deleted key is ignored', async () => { + const pages = [ + { + 1: '0x1', + 2: '0x2', + 3: '0x3', + 8: '0x8', + }, + { + 3: StorageValueKind.Deleted, + 7: '0x7', + }, + { + 1: StorageValueKind.Deleted, + 7: '0x77', + 8: StorageValueKind.Deleted, + 9: '0x9', + }, + { + 3: '0x33', + 4: '0x4', + }, + ] + + const prefix = '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' + const makeKey = (x: string) => prefix + '_' + Number(x).toString().padStart(2, '0') + + // build layers + const layers: StorageLayer[] = [] + for (const page of pages) { + const layer = new StorageLayer(layers[layers.length - 1]) + layer.setAll(Object.entries(page).map(([k, v]) => [makeKey(k), v] as [string, StorageValue])) + layers.push(layer) + } + + // last layer + expect(await layers[3].getKeysPaged(prefix, 100, prefix)).toEqual([ + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_02', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_03', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_04', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_07', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_09', + ]) + + expect( + await layers[3].getKeysPaged( + prefix, + 100, + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_05', + ), + ).toEqual([ + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_07', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_09', + ]) + + expect( + await layers[3].getKeysPaged( + prefix, + 100, + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_08', + ), + ).toEqual(['0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_09']) + + expect( + // previous layer + await layers[2].getKeysPaged(prefix, 100, prefix), + ).toEqual([ + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_02', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_07', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_09', + ]) + + expect( + // previous layer + await layers[1].getKeysPaged(prefix, 100, prefix), + ).toEqual([ + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_01', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_02', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_07', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_08', + ]) + + expect( + // previous layer + await layers[1].getKeysPaged( + prefix, + 100, + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_02', + ), + ).toEqual([ + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_07', + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff_08', + ]) + }) + + it('fuzz', async () => { + const oddPrefix = '0x1111111111111111111111111111111111111111111111111111111111111111' + const evenPrefix = '0x2222222222222222222222222222222222222222222222222222222222222222' + const makeKey = (x: number) => (x % 2 === 0 ? evenPrefix : oddPrefix) + '_' + x.toString().padStart(2, '0') + + // create some random keys + const pages: number[][] = [] + let p = Math.floor(Math.random() * 10) + 5 + while (p) { + p-- + const page: number[] = [] + let i = Math.floor(Math.random() * 10) + 5 + while (i) { + i-- + page.push(Math.floor(Math.random() * 30) + 1) + } + pages.push(page) + } + + // build layers + const layers: StorageLayer[] = [] + for (const page of pages) { + const layer = new StorageLayer(layers[layers.length - 1]) + layer.setAll(page.map((x) => [makeKey(x), '0x' + Number(x).toString(16)] as [string, StorageValue])) + layers.push(layer) + } + + const allKeys = pages + .flatMap((x) => x) + .reduce((acc, x) => { + if (acc.includes(x)) { + return acc + } + acc.push(x) + return acc + }, [] as number[]) + .sort((a, b) => a - b) + .map(makeKey) + + const oddKeys = await layers[layers.length - 1].getKeysPaged(oddPrefix, 100, oddPrefix) + expect(oddKeys, 'oddKeys').toEqual(allKeys.filter((x) => x.startsWith(oddPrefix))) + + const evenKeys = await layers[layers.length - 1].getKeysPaged(evenPrefix, 100, evenPrefix) + expect(evenKeys, 'evenKeys').toEqual(allKeys.filter((x) => x.startsWith(evenPrefix))) }) }) diff --git a/packages/core/src/blockchain/storage-layer.ts b/packages/core/src/blockchain/storage-layer.ts index c6fddc30..b75141ee 100644 --- a/packages/core/src/blockchain/storage-layer.ts +++ b/packages/core/src/blockchain/storage-layer.ts @@ -23,18 +23,14 @@ export interface StorageLayerProvider { * Get the value of a storage key. */ get(key: string, cache: boolean): Promise - /** - * Fold the storage layer into another layer. - */ - foldInto(into: StorageLayer): Promise - /** - * Fold the storage layer into the parent if it exists. - */ - fold(): Promise /** * Get paged storage keys. */ getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise + /** + * Find next storage key. + */ + findNextKey(prefix: string, startKey: string, knownBest?: string): Promise } export class RemoteStorageLayer implements StorageLayerProvider { @@ -63,10 +59,10 @@ export class RemoteStorageLayer implements StorageLayerProvider { return data ?? undefined } - async foldInto(_into: StorageLayer): Promise { - return this + async findNextKey(prefix: string, startKey: string, _knownBest?: string): Promise { + const keys = await this.getKeysPaged(prefix, 1, startKey) + return keys[0] } - async fold(): Promise {} async getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise { if (pageSize > BATCH_SIZE) throw new Error(`pageSize must be less or equal to ${BATCH_SIZE}`) @@ -76,7 +72,7 @@ export class RemoteStorageLayer implements StorageLayerProvider { const minPrefixLen = isChild ? CHILD_PREFIX_LENGTH : PREFIX_LENGTH // can't handle keyCache without prefix - if (prefix.length < minPrefixLen || startKey.length < minPrefixLen) { + if (prefix === startKey || prefix.length < minPrefixLen || startKey.length < minPrefixLen) { return this.#api.getKeysPaged(prefix, pageSize, startKey, this.#at) } @@ -209,136 +205,40 @@ export class StorageLayer implements StorageLayerProvider { } } - async foldInto(into: StorageLayer): Promise { - const newParent = await this.#parent?.foldInto(into) - - for (const deletedPrefix of this.#deletedPrefix) { - into.set(deletedPrefix, StorageValueKind.DeletedPrefix) - } - - for (const [key, value] of this.#store) { - into.set(key, await value) - } - - return newParent - } - - async fold(): Promise { - if (this.#parent) { - this.#parent = await this.#parent.foldInto(this) - } - } - - async getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise { - let parentFetchComplete = false - const parentFetchKeys = async (batchSize: number, startKey: string) => { - if (!this.#deletedPrefix.some((dp) => startKey.startsWith(dp))) { - const newKeys: string[] = [] - while (newKeys.length < batchSize) { - const remote = (await this.#parent?.getKeysPaged(prefix, batchSize, startKey)) ?? [] - if (remote.length) { - startKey = remote[remote.length - 1] - } - for (const key of remote) { - if (this.#store.get(key) === StorageValueKind.Deleted) { - continue - } - if (this.#deletedPrefix.some((dp) => key.startsWith(dp))) { - continue - } - newKeys.push(key) - } - if (remote.length < batchSize) { - parentFetchComplete = true - break - } - } - return newKeys - } else { - parentFetchComplete = true - return [] - } - } - - const res: string[] = [] - - const foundNextKey = (key: string) => { - // make sure keys are unique and start with the prefix - if (!res.includes(key) && key.startsWith(prefix)) { - res.push(key) - } + async findNextKey(prefix: string, startKey: string, knownBest?: string): Promise { + const maybeBest = this.#keys.find((key) => key.startsWith(prefix) && key > startKey) + if (!knownBest) { + knownBest = maybeBest + } else if (maybeBest && maybeBest < knownBest) { + knownBest = maybeBest } - - const iterLocalKeys = (prefix: string, startKey: string, includeFirst: boolean, endKey?: string) => { - let idx = this.#keys.findIndex((x) => x.startsWith(startKey)) - if (this.#keys[idx] !== startKey) { - idx = this.#keys.findIndex((x) => x.startsWith(prefix) && x > startKey) - const key = this.#keys[idx] - if (key) { - if (endKey && key >= endKey) { - return startKey - } - foundNextKey(key) - ++idx + if (this.#parent && !this.#deletedPrefix.some((dp) => dp === prefix)) { + const parentBest = await this.#parent.findNextKey(prefix, startKey, knownBest) + if (parentBest) { + if (!maybeBest) { + return parentBest + } else if (parentBest < maybeBest) { + return parentBest } } - if (idx !== -1) { - if (includeFirst) { - const key = this.#keys[idx] - if (key && key.startsWith(prefix) && key > startKey) { - foundNextKey(key) - } - } - while (res.length < pageSize) { - ++idx - const key: string = this.#keys[idx] - if (!key || !key.startsWith(prefix)) { - break - } - if (endKey && key >= endKey) { - break - } - foundNextKey(key) - } - return _.last(res) ?? startKey - } - return startKey } + return knownBest + } - if (prefix !== startKey && this.#keys.find((x) => x === startKey)) { - startKey = iterLocalKeys(prefix, startKey, false) + async getKeysPaged(prefix: string, pageSize: number, startKey: string): Promise { + if (!startKey || startKey === '0x') { + startKey = prefix } - // then iterate the parent keys - let keys = await parentFetchKeys(pageSize - res.length, startKey) - if (keys.length) { - let idx = 0 - while (res.length < pageSize) { - const key = keys[idx] - if (!key || !key.startsWith(prefix)) { - if (parentFetchComplete) { - break - } else { - keys = await parentFetchKeys(pageSize - res.length, _.last(keys)!) - continue - } - } - - const keyPosition = _.sortedIndex(this.#keys, key) - const localParentKey = this.#keys[keyPosition - 1] - if (localParentKey < key) { - startKey = iterLocalKeys(prefix, startKey, false, key) - } - - foundNextKey(key) - ++idx - } + const keys: string[] = [] + while (keys.length < pageSize) { + const next = await this.findNextKey(prefix, startKey, undefined) + if (!next) break + startKey = next + if ((await this.get(next, false)) == StorageValueKind.Deleted) continue + keys.push(next) } - if (res.length < pageSize) { - iterLocalKeys(prefix, startKey, prefix === startKey) - } - - return res + return keys } /** diff --git a/packages/db/package.json b/packages/db/package.json index ce2cb6a9..98e6aa5b 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@acala-network/chopsticks-db", - "version": "1.0.0", + "version": "1.0.1-1", "author": "Acala Developers ", "license": "Apache-2.0", "type": "module", diff --git a/packages/testing/package.json b/packages/testing/package.json index 89b5d94d..79abdc40 100644 --- a/packages/testing/package.json +++ b/packages/testing/package.json @@ -1,6 +1,6 @@ { "name": "@acala-network/chopsticks-testing", - "version": "1.0.0", + "version": "1.0.1-1", "author": "Acala Developers ", "license": "Apache-2.0", "type": "module", diff --git a/packages/utils/package.json b/packages/utils/package.json index 712f7b9a..72aaa9f8 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,6 +1,6 @@ { "name": "@acala-network/chopsticks-utils", - "version": "1.0.0", + "version": "1.0.1-1", "author": "Acala Developers ", "license": "Apache-2.0", "type": "module",