From e337727a170cb7398df20db508d983d9e63c9b5d Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Tue, 17 Oct 2023 16:40:22 +0200 Subject: [PATCH] use worker for wasm-executor (#445) * use worker for wasm-executor * fix path * fix buffer * worker terminate * run crowdloan.test only on CI --- executor/package.json | 2 +- .../scripts/{pack-wasm.js => pack-wasm.cjs} | 6 +- packages/core/package.json | 6 +- .../scripts/{pack-wasm.js => pack-wasm.cjs} | 12 +- packages/core/src/blockchain/block.ts | 4 +- packages/core/src/blockchain/index.ts | 4 +- .../inherent/parachain/validation-data.ts | 2 +- packages/core/src/genesis-provider.ts | 2 +- packages/core/src/index.ts | 2 +- packages/core/src/rpc/substrate/state.ts | 2 +- packages/core/src/utils/time-travel.ts | 2 +- .../__snapshots__/executor.test.ts.snap | 104 ++++++++++++++++++ .../wasm-executor/browser-wasm-executor.mjs | 37 +++++++ .../core/src/wasm-executor/browser-worker.ts | 17 +++ .../src/{ => wasm-executor}/executor.test.ts | 6 +- .../{executor.ts => wasm-executor/index.ts} | 91 ++++++++++----- .../src/wasm-executor/node-wasm-executor.mjs | 34 ++++++ .../core/src/wasm-executor/node-worker.ts | 17 +++ packages/e2e/src/build-block.test.ts | 6 +- packages/e2e/src/crowdloan.redeem.test.ts | 2 +- packages/web-test/index.html | 5 - packages/web-test/src/App.tsx | 4 +- .../{vite.config.js => vite.config.mjs} | 13 +-- yarn.lock | 8 ++ 24 files changed, 322 insertions(+), 66 deletions(-) rename executor/scripts/{pack-wasm.js => pack-wasm.cjs} (80%) rename packages/core/scripts/{pack-wasm.js => pack-wasm.cjs} (69%) create mode 100644 packages/core/src/wasm-executor/__snapshots__/executor.test.ts.snap create mode 100644 packages/core/src/wasm-executor/browser-wasm-executor.mjs create mode 100644 packages/core/src/wasm-executor/browser-worker.ts rename packages/core/src/{ => wasm-executor}/executor.test.ts (98%) rename packages/core/src/{executor.ts => wasm-executor/index.ts} (66%) create mode 100644 packages/core/src/wasm-executor/node-wasm-executor.mjs create mode 100644 packages/core/src/wasm-executor/node-worker.ts rename packages/web-test/{vite.config.js => vite.config.mjs} (53%) diff --git a/executor/package.json b/executor/package.json index eda289bf..47bbec4f 100644 --- a/executor/package.json +++ b/executor/package.json @@ -9,7 +9,7 @@ }, "scripts": { "clean": "rm -rf web node", - "build": "wasm-pack build --target web --out-dir browser; wasm-pack build --target nodejs --out-dir node; scripts/pack-wasm.js" + "build": "wasm-pack build --target web --out-dir browser; wasm-pack build --target nodejs --out-dir node; scripts/pack-wasm.cjs" }, "dependencies": { "@polkadot/wasm-util": "^7.2.2" diff --git a/executor/scripts/pack-wasm.js b/executor/scripts/pack-wasm.cjs similarity index 80% rename from executor/scripts/pack-wasm.js rename to executor/scripts/pack-wasm.cjs index f750d6f0..d9d56ff5 100755 --- a/executor/scripts/pack-wasm.js +++ b/executor/scripts/pack-wasm.cjs @@ -21,7 +21,11 @@ const WASM_BYTES = unzlibSync(base64Decode(BYTES, new Uint8Array(LEN_IN)), new U import wasmInit from "./chopsticks_executor.js"; const blob = new Blob([WASM_BYTES], { type: "application/wasm" }); -wasmInit(URL.createObjectURL(blob)); +export const wasmReady = wasmInit(URL.createObjectURL(blob)); export * from "./chopsticks_executor.js"; `); + +// replace wasm with empty file because it's not needed +// but we need to have it in the repo for the build to work +fs.writeFileSync(path.resolve(__dirname, `../browser/chopsticks_executor_bg.wasm`), '') diff --git a/packages/core/package.json b/packages/core/package.json index e3f942a5..45f4b093 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -5,8 +5,9 @@ "license": "Apache-2.0", "scripts": { "clean": "rm -rf lib tsconfig.tsbuildinfo", - "pack-wasm": "scripts/pack-wasm.js", - "build": "yarn pack-wasm; tsc -p ./tsconfig.json", + "pack-wasm": "scripts/pack-wasm.cjs", + "build": "yarn pack-wasm; tsc -p ./tsconfig.json; yarn copyfiles", + "copyfiles": "cp -r src/wasm-executor/*.mjs lib/wasm-executor/", "docs:prep": "typedoc" }, "dependencies": { @@ -14,6 +15,7 @@ "@polkadot/api": "^10.9.1", "@polkadot/util-crypto": "^12.3.2", "axios": "^1.5.1", + "comlink": "^4.4.1", "eventemitter3": "^5.0.1", "localforage": "^1.10.0", "lodash": "^4.17.21", diff --git a/packages/core/scripts/pack-wasm.js b/packages/core/scripts/pack-wasm.cjs similarity index 69% rename from packages/core/scripts/pack-wasm.js rename to packages/core/scripts/pack-wasm.cjs index 81dfcd6e..c2741af4 100755 --- a/packages/core/scripts/pack-wasm.js +++ b/packages/core/scripts/pack-wasm.cjs @@ -10,15 +10,15 @@ const compressed = Buffer.from(zlibSync(data, { level: 9 })) const base64 = compressed.toString('base64') console.log( - `*** Compressed WASM: in=${formatNumber(data.length)}, out=${formatNumber(compressed.length)}, opt=${( - (100 * compressed.length) / - data.length - ).toFixed(2)}%, base64=${formatNumber(base64.length)}`, + `*** Compressed WASM: in=${formatNumber(data.length)}, out=${formatNumber(compressed.length)}, opt=${( + (100 * compressed.length) / + data.length + ).toFixed(2)}%, base64=${formatNumber(base64.length)}`, ) fs.writeFileSync( - path.resolve(__dirname, `../src/db/sql-wasm.ts`), - `// Auto-generated file, do not edit by hand + path.resolve(__dirname, `../src/db/sql-wasm.ts`), + `// Auto-generated file, do not edit by hand const LEN_IN = ${compressed.length} const LEN_OUT = ${data.length} const BYTES = diff --git a/packages/core/src/blockchain/block.ts b/packages/core/src/blockchain/block.ts index 9d394cf6..58fedc1c 100644 --- a/packages/core/src/blockchain/block.ts +++ b/packages/core/src/blockchain/block.ts @@ -11,8 +11,8 @@ import { Blockchain } from '.' import { RemoteStorageLayer, StorageLayer, StorageLayerProvider, StorageValue, StorageValueKind } from './storage-layer' import { compactHex } from '../utils' import { defaultLogger } from '../logger' -import { getRuntimeVersion, runTask, taskHandler } from '../executor' -import type { RuntimeVersion } from '../executor' +import { getRuntimeVersion, runTask, taskHandler } from '../wasm-executor' +import type { RuntimeVersion } from '../wasm-executor' export type TaskCallResponse = { result: HexString diff --git a/packages/core/src/blockchain/index.ts b/packages/core/src/blockchain/index.ts index 80f526d3..ee1a786b 100644 --- a/packages/core/src/blockchain/index.ts +++ b/packages/core/src/blockchain/index.ts @@ -17,6 +17,7 @@ import { StorageValue } from './storage-layer' import { compactHex } from '../utils' import { defaultLogger } from '../logger' import { dryRunExtrinsic, dryRunInherents } from './block-builder' +import { releaseWorker } from '../wasm-executor' const logger = defaultLogger.child({ name: 'blockchain' }) @@ -510,9 +511,10 @@ export class Blockchain { } /** - * Close the db. + * Close the db and release worker. */ async close() { + await releaseWorker() await this.db?.destroy() } } diff --git a/packages/core/src/blockchain/inherent/parachain/validation-data.ts b/packages/core/src/blockchain/inherent/parachain/validation-data.ts index 82810960..f8d56947 100644 --- a/packages/core/src/blockchain/inherent/parachain/validation-data.ts +++ b/packages/core/src/blockchain/inherent/parachain/validation-data.ts @@ -17,7 +17,7 @@ import { } from '../../../utils/proof' import { blake2AsHex, blake2AsU8a } from '@polkadot/util-crypto' import { compactHex, getParaId } from '../../../utils' -import { createProof, decodeProof } from '../../../executor' +import { createProof, decodeProof } from '../../../wasm-executor' const MOCK_VALIDATION_DATA = { validationData: { diff --git a/packages/core/src/genesis-provider.ts b/packages/core/src/genesis-provider.ts index fb2acd75..a2828a26 100644 --- a/packages/core/src/genesis-provider.ts +++ b/packages/core/src/genesis-provider.ts @@ -10,7 +10,7 @@ import { import axios from 'axios' import { Genesis, genesisSchema } from './schema' -import { JsCallback, calculateStateRoot, emptyTaskHandler } from './executor' +import { JsCallback, calculateStateRoot, emptyTaskHandler } from './wasm-executor' import { isUrl } from './utils' export class GenesisProvider implements ProviderInterface { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e524ce6c..df529a0f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -15,7 +15,7 @@ export * from './blockchain/txpool' export * from './blockchain/storage-layer' export * from './blockchain/head-state' export * from './utils' -export * from './executor' +export * from './wasm-executor' export * from './schema' export * from './xcm' export * from './setup' diff --git a/packages/core/src/rpc/substrate/state.ts b/packages/core/src/rpc/substrate/state.ts index 074bdae8..d3ca3551 100644 --- a/packages/core/src/rpc/substrate/state.ts +++ b/packages/core/src/rpc/substrate/state.ts @@ -2,7 +2,7 @@ import { Block } from '../../blockchain/block' import { HexString } from '@polkadot/util/types' import { Handler, ResponseError } from '../shared' -import { RuntimeVersion } from '../../executor' +import { RuntimeVersion } from '../../wasm-executor' import { defaultLogger } from '../../logger' import { isPrefixedChildKey, prefixedChildKey, stripChildPrefix } from '../../utils' diff --git a/packages/core/src/utils/time-travel.ts b/packages/core/src/utils/time-travel.ts index 51c6eaa3..dd6ae283 100644 --- a/packages/core/src/utils/time-travel.ts +++ b/packages/core/src/utils/time-travel.ts @@ -4,7 +4,7 @@ import { Slot } from '@polkadot/types/interfaces' import { Blockchain } from '../blockchain' import { compactHex } from '.' -import { getAuraSlotDuration } from '../executor' +import { getAuraSlotDuration } from '../wasm-executor' import { setStorage } from './set-storage' export const getCurrentSlot = async (chain: Blockchain) => { diff --git a/packages/core/src/wasm-executor/__snapshots__/executor.test.ts.snap b/packages/core/src/wasm-executor/__snapshots__/executor.test.ts.snap new file mode 100644 index 00000000..4b9d6cef --- /dev/null +++ b/packages/core/src/wasm-executor/__snapshots__/executor.test.ts.snap @@ -0,0 +1,104 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`wasm > decode & create proof works 1`] = ` +{ + "0x06de3d8a54d27e44a9d5ce189618f22db4b49d95320d9021994c850f25b8e385": "0x00005000005000000a00000000c8000000c800000a0000000a000000c8000000640000000000500000c800000700e8764817020040010a0000000000000000c0220fca950300000000000000000000c0220fca9503000000000000000000e8030000009001000a00000000000000009001008070000000000000000000000a000000050000000500000001000000010500000001c800000006000000580200005802000002000000280000000000000002000000010000000700c817a8040200400101020000000f000000", + "0x1cb6f36e027abb2091cfb5110ab5087f06155b3cd9a8c9e5e9a23fd5dc13a5ed": "0x5d49a91000000000", + "0x1cb6f36e027abb2091cfb5110ab5087f38316cbf8fa0da822a20ac1c55bf1be3": "0xa262000000000000", + "0x1cb6f36e027abb2091cfb5110ab5087f7a414cb008e0e61e46722aa60abdd672": "0x2527aaef690b1334e4ec1bd567ea892286dc1265510ad57d74dd8628d4c47ad4", + "0x1cb6f36e027abb2091cfb5110ab5087f7ce678799d3eff024253b90e84927cc6": "0xcc322cfedfa300d9bec6649d65f16273941d0e2f90a4c0d7173dd7c95bb02597", + "0x1cb6f36e027abb2091cfb5110ab5087fd077dfdb8adb10f78f10a5df8742c545": "0x01083c38641effe2a1c267fb16654d73afd072e5bbbb58a3cee00b6f07858d7868", + "0x63f78c98723ddc9073523ef3beefda0c4d7fefc408aac59dbfe80a72ac8e3ce5b6ff6f7d467b87a9e8030000": "0xf217a51bf18a93d44906f8a3dbd8f60acb609b03c9ade7e38532e3f5f18f70e8", + "0x6a0da05ca59913bc38a8630590f2627c1d3719f5b0b12c7105c073c507445948b6ff6f7d467b87a9e8030000": "0x08d007000049080000", + "0x6a0da05ca59913bc38a8630590f2627cf12b746dcf32e843354583c9702cc020b6ff6f7d467b87a9e8030000": "0x08d007000049080000", + "0xcd710b30bd2eab0352ddcc26417aa1949e94c040f5e73d9b7addd6cb603d15d3b6ff6f7d467b87a9e8030000": undefined, +} +`; + +exports[`wasm > decode & create proof works 2`] = ` +{ + "chainAvailabilityPeriod": 5, + "codeRetentionPeriod": 0, + "disputeConclusionByTimeOutPeriod": 2, + "disputeMaxSpamSlots": 600, + "disputePeriod": 6, + "disputePostConclusionAcceptancePeriod": 600, + "groupRotationFrequency": 5, + "hrmpChannelMaxCapacity": 102400, + "hrmpChannelMaxMessageSize": 28800, + "hrmpChannelMaxTotalSize": 10, + "hrmpMaxMessageNumPerCandidate": 10, + "hrmpMaxParachainInboundChannels": 0, + "hrmpMaxParachainOutboundChannels": 10, + "hrmpMaxParathreadInboundChannels": 102400, + "hrmpMaxParathreadOutboundChannels": 0, + "hrmpOpenRequestTtl": 253935616, + "hrmpRecipientDeposit": "0x000003e80000000000000000000395ca", + "hrmpSenderDeposit": "0x0f22c0000000000000000000000395ca", + "maxCodeSize": 5242880, + "maxDownwardMessageSize": 51200, + "maxHeadDataSize": 20480, + "maxPovSize": 5242880, + "maxUpwardMessageNumPerCandidate": 10, + "maxUpwardMessageSize": 51200, + "maxUpwardQueueCount": 10, + "maxUpwardQueueSize": 51200, + "maxValidators": 200, + "maxValidatorsPerCore": null, + "nDelayTranches": 0, + "neededApprovals": 1, + "noShowSlots": 40, + "parathreadCores": 0, + "parathreadRetries": 10, + "preferredDispatchableUpwardMessagesStepWeight": { + "proofSize": 5242880, + "refTime": 100000000000, + }, + "relayVrfModuloSamples": 398983175, + "schedulingLookahead": 1281, + "threadAvailabilityPeriod": 1, + "validationUpgradeDelay": 100, + "validationUpgradeFrequency": 200, + "zerothDelayTrancheWidth": 2, +} +`; + +exports[`wasm > decode & create proof works 3`] = `"0x7db04060b46e67b4480c3cf9a1be364eca21a2972ce03f35b9d69d0015db1549"`; + +exports[`wasm > decode & create proof works 4`] = ` +[ + "0x9f012b746dcf32e843354583c9702cc02040884c5703f5a4efb16ffa83d00700001404e80300005c5706ff6f7d467b87a9e80300002408d0070000490800004c570c7327a2a48bf2b1490800001404e8030000", + "0x9f0cb6f36e027abb2091cfb5110ab5087f9960685f06155b3cd9a8c9e5e9a23fd5dc13a5ed205d49a91000000000685f08316cbf8fa0da822a20ac1c55bf1be320a262000000000000505f0e7b9012096b41c4eb3aaf947f6ea42908000080b06947b43b690f37d4acce3df0061e956e4b4249d58b80519935833c2c4eb0e680e627529ec0b70df9d4562816343d3fd0d30abe8b2b0c8d812a25d94a55974a34685f090e2fbf2d792cb324bffa9427fe1f0e20705be100ae5de100", + "0x9f0d710b30bd2eab0352ddcc26417aa19440824c5f03c716fb8fff3de61a883bb76adb34a2040080107a2888b632f818a19adeb1ccd4ddc7d677ea02956d755f678536a83a25f35e4c5f0dad6eef5c4b1c68eaa71ea17a02d9de0400", + "0x8000c080a703489351d812e54a745e221a7469907a21479e2cd8a9776ae340fdda898982485e4993f016e2d2f8e5f43be7bb2594860400", + "0x9f05207f03cfdce586301014700e2c2593d040505f0e7b9012096b41c4eb3aaf947f6ea429080100685f0d9ef3b78afddab7f5c7142131132ad4200700000000000000585f02275f64c354954352b71eea39cfaca210070000004c5f0ec2d17a76153ff51817f12d9cfc3c7f0400", + "0x800804806d6cd404605d5c2481bf43cb9403a6c8d6da004318e39fe1ff31c9f50042febd80929fccaf143551e3cee678117c06e6b8ebd6e3443214a8cbde73fb82d89c7111", + "0x5e414cb008e0e61e46722aa60abdd672802527aaef690b1334e4ec1bd567ea892286dc1265510ad57d74dd8628d4c47ad4", + "0x9f06de3d8a54d27e44a9d5ce189618f22d3008505f0e7b9012096b41c4eb3aaf947f6ea4290804004c5f03b4123b2e186e07fb7bad5dda5f55c004008077d0e6c6873fc637a60aa06503ed4ffbd0bfdd2b5d2f012343a72a303614afae", + "0x767fefc408aac59dbfe80a72ac8e3ce5b6ff6f7d467b87a9e803000080f217a51bf18a93d44906f8a3dbd8f60acb609b03c9ade7e38532e3f5f18f70e8", + "0x5f04b49d95320d9021994c850f25b8e3852d0300005000005000000a00000000c8000000c800000a0000000a000000c8000000640000000000500000c800000700e8764817020040010a0000000000000000c0220fca950300000000000000000000c0220fca9503000000000000000000e8030000009001000a00000000000000009001008070000000000000000000000a000000050000000500000001000000010500000001c800000006000000580200005802000002000000280000000000000002000000010000000700c817a8040200400101020000000f000000", + "0x5ee678799d3eff024253b90e84927cc680cc322cfedfa300d9bec6649d65f16273941d0e2f90a4c0d7173dd7c95bb02597", + "0x5f0077dfdb8adb10f78f10a5df8742c5458401083c38641effe2a1c267fb16654d73afd072e5bbbb58a3cee00b6f07858d7868", + "0x8043908017950ddfb68b41a6e0eb68483384a2642afdb241fd0d7fe021e5b9f20b6f275c80c94292198e0c8b47b9aee514c18d22879f4cb9f6bc9e5bb4e38594917f6f829680a3fd676e9772de52c2aa8efb3646ed1d6268894f406f415964305fc6352f0e7b804bd34dd0ef7e09359688e6df7b75dc510bb89d37ef08be2432302c2efef125bb80c31f960bc177852c1ff08ad7f576cd2955a72c9d8df1fe35cc0d98a49d730a6f", + "0x9f0f78c98723ddc9073523ef3beefda0c4006080466494236add4d65df86a49c798c1e67a9bbf772792e80988ba6923c109d59f44c5e7b9012096b41c4eb3aaf947f6ea429080000", + "0x9f0d3719f5b0b12c7105c073c50744594840884c5703f5a4efb16ffa83d00700001404e80300005c5706ff6f7d467b87a9e80300002408d0070000490800004c570c7327a2a48bf2b1490800001404e8030000", + "0x9e0da05ca59913bc38a8630590f2627c168080c208804fb9b3b0d5906eb36f941e821db3f79cb3210ca8aa4c0560dcc27980534c5f0a351b6a99a5b21324516e668bb86a570400505f0e7b9012096b41c4eb3aaf947f6ea42908000080faa229c2dbfea7ecf83e772ea2bc6e7bf8f77ff9b247e7ac953be64c15c498d6", + "0x9e94c040f5e73d9b7addd6cb603d15d308083c5701275d2a59e7bf2be507000004013c5706ff6f7d467b87a9e80300000401", + "0x80001480d1c5dfd27a348dfe6d0bbba95e4c43019bd078a4bf616f6a982a6b7eab2c932080c5c3fb9143839ec1e62ace0d5f5a99d1bc410e3475c3220c568c828ac1ec31db", +] +`; + +exports[`wasm > decode & create proof works 5`] = ` +{ + "0x06de3d8a54d27e44a9d5ce189618f22db4b49d95320d9021994c850f25b8e385": "0x00005000005000000a00000000c8000000c800000a0000000a000000c8000000640000000000500000c800000700e8764817020040010a0000000000000000c0220fca950300000000000000000000c0220fca9503000000000000000000e8030000009001000a00000000000000009001008070000000000000000000000a000000050000000500000001000000010500000001c800000006000000580200005802000002000000280000000000000002000000010000000700c817a8040200400101020000000f000000", + "0x1cb6f36e027abb2091cfb5110ab5087f06155b3cd9a8c9e5e9a23fd5dc13a5ed": "0x5d49a91000000000", + "0x1cb6f36e027abb2091cfb5110ab5087f38316cbf8fa0da822a20ac1c55bf1be3": "0xa262000000000000", + "0x1cb6f36e027abb2091cfb5110ab5087f7a414cb008e0e61e46722aa60abdd672": "0x2527aaef690b1334e4ec1bd567ea892286dc1265510ad57d74dd8628d4c47ad4", + "0x1cb6f36e027abb2091cfb5110ab5087f7ce678799d3eff024253b90e84927cc6": "0xcc322cfedfa300d9bec6649d65f16273941d0e2f90a4c0d7173dd7c95bb02597", + "0x1cb6f36e027abb2091cfb5110ab5087fd077dfdb8adb10f78f10a5df8742c545": "0x01083c38641effe2a1c267fb16654d73afd072e5bbbb58a3cee00b6f07858d7868", + "0x63f78c98723ddc9073523ef3beefda0c4d7fefc408aac59dbfe80a72ac8e3ce5b6ff6f7d467b87a9e8030000": "0xf217a51bf18a93d44906f8a3dbd8f60acb609b03c9ade7e38532e3f5f18f70e8", + "0x6a0da05ca59913bc38a8630590f2627c1d3719f5b0b12c7105c073c507445948b6ff6f7d467b87a9e8030000": "0x08d007000049080000", + "0x6a0da05ca59913bc38a8630590f2627cf12b746dcf32e843354583c9702cc020b6ff6f7d467b87a9e8030000": "0x08d007000049080000", + "0xcd710b30bd2eab0352ddcc26417aa1949e94c040f5e73d9b7addd6cb603d15d3b6ff6f7d467b87a9e8030000": "0x01", +} +`; diff --git a/packages/core/src/wasm-executor/browser-wasm-executor.mjs b/packages/core/src/wasm-executor/browser-wasm-executor.mjs new file mode 100644 index 00000000..b20adf72 --- /dev/null +++ b/packages/core/src/wasm-executor/browser-wasm-executor.mjs @@ -0,0 +1,37 @@ +import * as Comlink from 'comlink' +import * as pkg from '@acala-network/chopsticks-executor' + +const getRuntimeVersion = async (code) => { + await pkg.wasmReady + return pkg.get_runtime_version(code) +} + +// trie_version: 0 for old trie, 1 for new trie +const calculateStateRoot = async (entries, trie_version) => { + await pkg.wasmReady + return pkg.calculate_state_root(entries, trie_version) +} + +const decodeProof = async (trieRootHash, keys, nodes) => { + await pkg.wasmReady + const decoded = await pkg.decode_proof(trieRootHash, keys, nodes) + return decoded.reduce((accum, [key, value]) => { + accum[key] = value + return accum + }, {}) +} + +const createProof = async (nodes, entries) => { + await pkg.wasmReady + const result = await pkg.create_proof(nodes, entries) + return { trieRootHash: result[0], nodes: result[1] } +} + +const runTask = async (task, callback) => { + await pkg.wasmReady + return pkg.run_task(task, callback, 'info') +} + +const wasmExecutor = { runTask, getRuntimeVersion, calculateStateRoot, createProof, decodeProof } + +Comlink.expose(wasmExecutor) diff --git a/packages/core/src/wasm-executor/browser-worker.ts b/packages/core/src/wasm-executor/browser-worker.ts new file mode 100644 index 00000000..d92d1c86 --- /dev/null +++ b/packages/core/src/wasm-executor/browser-worker.ts @@ -0,0 +1,17 @@ +import { wrap } from 'comlink' +import type { WasmExecutor } from '.' + +export const startWorker = async () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const worker = new Worker(new URL('browser-wasm-executor.mjs', import.meta.url), { + type: 'module', + name: 'chopsticks-wasm-executor', + }) + return { + remote: wrap(worker), + terminate: async () => { + worker.terminate() + }, + } +} diff --git a/packages/core/src/executor.test.ts b/packages/core/src/wasm-executor/executor.test.ts similarity index 98% rename from packages/core/src/executor.test.ts rename to packages/core/src/wasm-executor/executor.test.ts index 7ff4d5d6..34d27b39 100644 --- a/packages/core/src/executor.test.ts +++ b/packages/core/src/wasm-executor/executor.test.ts @@ -10,11 +10,11 @@ import { hrmpEgressChannelIndex, hrmpIngressChannelIndex, upgradeGoAheadSignal, -} from './utils/proof' -import { calculateStateRoot, createProof, decodeProof, getAuraSlotDuration, getRuntimeVersion } from './executor' +} from '../utils/proof' +import { calculateStateRoot, createProof, decodeProof, getAuraSlotDuration, getRuntimeVersion } from '.' const getCode = () => { - const code = String(readFileSync(path.join(__dirname, '../../e2e/blobs/acala-runtime-2101.txt'))).trim() + const code = String(readFileSync(path.join(__dirname, '../../../e2e/blobs/acala-runtime-2101.txt'))).trim() expect(code.length).toBeGreaterThan(2) return code as HexString } diff --git a/packages/core/src/executor.ts b/packages/core/src/wasm-executor/index.ts similarity index 66% rename from packages/core/src/executor.ts rename to packages/core/src/wasm-executor/index.ts index 02f45de2..9f73a027 100644 --- a/packages/core/src/executor.ts +++ b/packages/core/src/wasm-executor/index.ts @@ -1,22 +1,16 @@ +import * as Comlink from 'comlink' import { HexString } from '@polkadot/util/types' +import { Registry } from '@polkadot/types-codec/types' import { hexToString, hexToU8a } from '@polkadot/util' import { randomAsHex } from '@polkadot/util-crypto' - -import { Block } from './blockchain/block' -import { PREFIX_LENGTH } from './utils/key-cache' -import { Registry } from '@polkadot/types-codec/types' -import { - calculate_state_root, - create_proof, - decode_proof, - get_runtime_version, - run_task, -} from '@acala-network/chopsticks-executor' -import { defaultLogger, truncate } from './logger' -import { stripChildPrefix } from './utils' import _ from 'lodash' -import type { JsCallback } from '@acala-network/chopsticks-executor' +import { Block } from '../blockchain/block' +import { PREFIX_LENGTH } from '../utils/key-cache' +import { defaultLogger, truncate } from '../logger' +import { stripChildPrefix } from '../utils' + +import type { JsCallback } from '@acala-network/chopsticks-executor' export { JsCallback } export type RuntimeVersion = { @@ -30,10 +24,49 @@ export type RuntimeVersion = { stateVersion: number } +export interface WasmExecutor { + getRuntimeVersion: (code: HexString) => Promise + calculateStateRoot: (entries: [HexString, HexString][], trie_version: number) => Promise + createProof: ( + nodes: HexString[], + entries: [HexString, HexString | null][], + ) => Promise<{ + trieRootHash: `0x${string}` + nodes: `0x${string}`[] + }> + decodeProof: ( + trieRootHash: HexString, + keys: HexString[], + nodes: HexString[], + ) => Promise> + runTask: ( + task: { + wasm: HexString + calls: [string, HexString[]][] + mockSignatureHost: boolean + allowUnresolvedImports: boolean + runtimeLogLevel: number + }, + callback?: JsCallback, + ) => Promise +} + const logger = defaultLogger.child({ name: 'executor' }) +let __executor_worker: Promise<{ remote: Comlink.Remote; terminate: () => Promise }> | undefined +const getWorker = async () => { + if (__executor_worker) return __executor_worker + if (typeof Worker !== 'undefined') { + __executor_worker = import('./browser-worker').then(({ startWorker }) => startWorker()) + } else { + __executor_worker = import('./node-worker').then(({ startWorker }) => startWorker()) + } + return __executor_worker +} + export const getRuntimeVersion = async (code: HexString): Promise => { - return get_runtime_version(code).then((version) => { + const worker = await getWorker() + return worker.remote.getRuntimeVersion(code).then((version) => { version.specName = hexToString(version.specName) version.implName = hexToString(version.implName) return version @@ -45,23 +78,18 @@ export const calculateStateRoot = async ( entries: [HexString, HexString][], trie_version: number, ): Promise => { - return calculate_state_root(entries, trie_version) + const worker = await getWorker() + return worker.remote.calculateStateRoot(entries, trie_version) } export const decodeProof = async (trieRootHash: HexString, keys: HexString[], nodes: HexString[]) => { - const decoded: [HexString, HexString | null][] = await decode_proof(trieRootHash, keys, nodes) - return decoded.reduce( - (accum, [key, value]) => { - accum[key] = value - return accum - }, - {} as Record, - ) + const worker = await getWorker() + return worker.remote.decodeProof(trieRootHash, keys, nodes) } export const createProof = async (nodes: HexString[], entries: [HexString, HexString | null][]) => { - const result = await create_proof(nodes, entries) - return { trieRootHash: result[0] as HexString, nodes: result[1] as HexString[] } + const worker = await getWorker() + return worker.remote.createProof(nodes, entries) } export const runTask = async ( @@ -74,8 +102,9 @@ export const runTask = async ( }, callback: JsCallback = emptyTaskHandler, ) => { + const worker = await getWorker() logger.trace(truncate(task), 'taskRun') - const response = await run_task(task, callback, typeof process === 'object' ? process.env.RUST_LOG : 'info') + const response = await worker.remote.runTask(task, Comlink.proxy(callback)) if (response.Call) { logger.trace(truncate(response.Call), 'taskResponse') } else { @@ -162,3 +191,11 @@ export const getAuraSlotDuration = _.memoize(async (wasm: HexString, registry: R const slotDuration = registry.createType('u64', hexToU8a(result.Call.result)).toNumber() return slotDuration }) + +export const releaseWorker = async () => { + if (!__executor_worker) return + const executor = await __executor_worker + executor.remote[Comlink.releaseProxy]() + await executor.terminate() + __executor_worker = undefined +} diff --git a/packages/core/src/wasm-executor/node-wasm-executor.mjs b/packages/core/src/wasm-executor/node-wasm-executor.mjs new file mode 100644 index 00000000..e272740f --- /dev/null +++ b/packages/core/src/wasm-executor/node-wasm-executor.mjs @@ -0,0 +1,34 @@ +import * as Comlink from 'comlink' +import * as pkg from '@acala-network/chopsticks-executor' +import { parentPort } from 'node:worker_threads' +import nodeEndpoint from 'comlink/dist/umd/node-adapter.js' + +const getRuntimeVersion = async (code) => { + return pkg.get_runtime_version(code) +} + +// trie_version: 0 for old trie, 1 for new trie +const calculateStateRoot = async (entries, trie_version) => { + return pkg.calculate_state_root(entries, trie_version) +} + +const decodeProof = async (trieRootHash, keys, nodes) => { + const decoded = await pkg.decode_proof(trieRootHash, keys, nodes) + return decoded.reduce((accum, [key, value]) => { + accum[key] = value + return accum + }, {}) +} + +const createProof = async (nodes, entries) => { + const result = await pkg.create_proof(nodes, entries) + return { trieRootHash: result[0], nodes: result[1] } +} + +const runTask = async (task, callback) => { + return pkg.run_task(task, callback, process.env.RUST_LOG) +} + +export const wasmExecutor = { runTask, getRuntimeVersion, calculateStateRoot, createProof, decodeProof } + +Comlink.expose(wasmExecutor, nodeEndpoint(parentPort)) diff --git a/packages/core/src/wasm-executor/node-worker.ts b/packages/core/src/wasm-executor/node-worker.ts new file mode 100644 index 00000000..83f0d126 --- /dev/null +++ b/packages/core/src/wasm-executor/node-worker.ts @@ -0,0 +1,17 @@ +import { wrap } from 'comlink' +import nodeEndpoint from 'comlink/dist/umd/node-adapter.js' +import threads from 'node:worker_threads' +import url from 'node:url' +import type { WasmExecutor } from '.' + +export const startWorker = async () => { + const worker = new threads.Worker(url.resolve(__filename, 'node-wasm-executor.mjs'), { + name: 'chopsticks-wasm-executor', + }) + return { + remote: wrap(nodeEndpoint(worker)), + terminate: async () => { + await worker.terminate() + }, + } +} diff --git a/packages/e2e/src/build-block.test.ts b/packages/e2e/src/build-block.test.ts index 1f1c631e..27c496ea 100644 --- a/packages/e2e/src/build-block.test.ts +++ b/packages/e2e/src/build-block.test.ts @@ -12,7 +12,7 @@ const KUSAMA_STORAGE = { }, } -describe.each([ +describe.runIf(process.env.CI).each([ { chain: 'Polkadot', endpoint: 'wss://rpc.polkadot.io' }, { chain: 'Statemint', endpoint: 'wss://statemint-rpc.polkadot.io' }, { chain: 'Polkadot Collectives', endpoint: 'wss://polkadot-collectives-rpc.polkadot.io' }, @@ -32,7 +32,7 @@ describe.each([ await teardownAll() }) - it.runIf(process.env.CI)('build blocks', async () => { + it('build blocks', async () => { const { chain, ws, teardown } = await setup() storage && (await ws.send('dev_setStorage', [storage])) const blockNumber = chain.head.number @@ -41,7 +41,7 @@ describe.each([ await teardown() }) - it.runIf(process.env.CI)('build block using unsafeBlockHeight', async () => { + it('build block using unsafeBlockHeight', async () => { const { chain, ws, teardown } = await setup() storage && (await ws.send('dev_setStorage', [storage])) const blockNumber = chain.head.number diff --git a/packages/e2e/src/crowdloan.redeem.test.ts b/packages/e2e/src/crowdloan.redeem.test.ts index aab07f2e..a11384ff 100644 --- a/packages/e2e/src/crowdloan.redeem.test.ts +++ b/packages/e2e/src/crowdloan.redeem.test.ts @@ -3,7 +3,7 @@ import { testingPairs } from '@acala-network/chopsticks-testing' import networks from './networks' -describe('Polkadot Crowdloan Refund', async () => { +describe.runIf(process.env.CI)('Polkadot Crowdloan Refund', async () => { const { alice } = testingPairs() const { api, dev, teardown } = await networks.polkadot({ blockNumber: 17700000, timeout: 400_000 }) diff --git a/packages/web-test/index.html b/packages/web-test/index.html index 501ee8cc..6e40d51f 100644 --- a/packages/web-test/index.html +++ b/packages/web-test/index.html @@ -5,11 +5,6 @@ - -
diff --git a/packages/web-test/src/App.tsx b/packages/web-test/src/App.tsx index 6e7650d9..c97917f7 100644 --- a/packages/web-test/src/App.tsx +++ b/packages/web-test/src/App.tsx @@ -15,6 +15,8 @@ import { setStorage, setup } from '@acala-network/chopsticks-core' import { styled } from '@mui/system' import { useEffect, useState } from 'react' import type { SetupOptions } from '@acala-network/chopsticks-core' +import { Buffer } from 'buffer' +window.Buffer = Buffer const DocsLink = styled('a')` position: absolute; @@ -249,7 +251,7 @@ function App() { label="Extrinsic" value={extrinsic} multiline - maxRows={4} + rows={3} sx={{ mt: 1 }} onChange={(e) => { setExtrinsic(e.target.value) diff --git a/packages/web-test/vite.config.js b/packages/web-test/vite.config.mjs similarity index 53% rename from packages/web-test/vite.config.js rename to packages/web-test/vite.config.mjs index 91f69007..f8a090a8 100644 --- a/packages/web-test/vite.config.js +++ b/packages/web-test/vite.config.mjs @@ -3,12 +3,9 @@ import react from '@vitejs/plugin-react' import tsconfigPaths from 'vite-tsconfig-paths' export default defineConfig({ - plugins: [tsconfigPaths(), react()], - base: '/chopsticks/', - build: { - outDir: '../../dist', - }, - worker: { - format: 'es', - }, + plugins: [tsconfigPaths(), react()], + base: '/chopsticks/', + build: { + outDir: '../../dist', + }, }) diff --git a/yarn.lock b/yarn.lock index d2f6f074..971bf291 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22,6 +22,7 @@ __metadata: "@types/lodash": ^4.14.199 "@types/sql.js": ^1.4.4 axios: ^1.5.1 + comlink: ^4.4.1 eventemitter3: ^5.0.1 fflate: ^0.8.0 localforage: ^1.10.0 @@ -3740,6 +3741,13 @@ __metadata: languageName: node linkType: hard +"comlink@npm:^4.4.1": + version: 4.4.1 + resolution: "comlink@npm:4.4.1" + checksum: 16d58a8f590087fc45432e31d6c138308dfd4b75b89aec0b7f7bb97ad33d810381bd2b1e608a1fb2cf05979af9cbfcdcaf1715996d5fcf77aeb013b6da3260af + languageName: node + linkType: hard + "concat-map@npm:0.0.1": version: 0.0.1 resolution: "concat-map@npm:0.0.1"