use indexedDB instead of sqljs (#477)

* add idb for browser * add test * fix test * remove extra package * remove package * remove localforage * using cursor * add more tests * fix test * fix tests * fix lint * fix keyValue key * fix * reduce key value storage size * small fix * undo comment --------- Co-authored-by: Ermal Kaleci <ermalkaleci@gmail.com>
AcalaNetwork · Oct 29, 2023 · a38d901 · a38d901
1 parent 889fb52
commit a38d901
Show file tree

Hide file tree

Showing 8 changed files with 175 additions and 141 deletions.
diff --git a/packages/db/package.json b/packages/db/package.json
@@ -5,21 +5,15 @@
 	"license": "Apache-2.0",
 	"scripts": {
 		"clean": "rm -rf lib tsconfig.tsbuildinfo",
-		"pack-wasm": "scripts/pack-wasm.cjs",
-		"build": "yarn pack-wasm; tsc -p ./tsconfig.json"
+		"build": "tsc -p ./tsconfig.json"
 	},
 	"dependencies": {
 		"@acala-network/chopsticks-core": "workspace:*",
-		"@polkadot/util": "^12.5.1",
-		"@polkadot/wasm-util": "^7.2.2",
-		"localforage": "^1.10.0",
-		"sql.js": "^1.8.0",
+		"idb": "^7.1.1",
 		"sqlite3": "^5.1.6",
 		"typeorm": "^0.3.17"
 	},
 	"devDependencies": {
-		"@types/sql.js": "^1.4.4",
-		"fflate": "^0.8.0",
 		"typescript": "^5.1.6"
 	},
 	"files": [

diff --git a/packages/db/scripts/pack-wasm.cjs b/packages/db/scripts/pack-wasm.cjs
diff --git a/packages/db/src/browser.ts b/packages/db/src/browser.ts
@@ -1,13 +1,81 @@
-import { DataSource } from 'typeorm'
+import { BlockEntry, Database, KeyValueEntry } from '@acala-network/chopsticks-core'
+import { DBSchema, IDBPDatabase, openDB } from 'idb'
 
-import { BaseSqlDatabase } from './base-sql'
-import { openDb } from './db/browser'
-
-export class SqljsDatabase extends BaseSqlDatabase {
-  datasource: Promise<DataSource>
+interface Schema extends DBSchema {
+  keyValue: {
+    key: string
+    value: string | null
+  }
+  block: {
+    key: string
+    value: BlockEntry
+    indexes: { byNumber: number }
+  }
+}
+export class IdbDatabase implements Database {
+  datasource: Promise<IDBPDatabase<Schema>>
 
   constructor(location: string) {
-    super()
-    this.datasource = openDb(location)
+    this.datasource = openDB<Schema>(location, 1, {
+      upgrade(db) {
+        db.createObjectStore('keyValue')
+        const blockStore = db.createObjectStore('block', { keyPath: 'hash' })
+        blockStore.createIndex('byNumber', 'number')
+      },
+    })
+  }
+
+  async close(): Promise<void> {
+    const db = await this.datasource
+    db.close()
+  }
+
+  async saveBlock(block: BlockEntry): Promise<void> {
+    const db = await this.datasource
+    const tx = db.transaction(['block'], 'readwrite')
+    const store = tx.objectStore('block')
+    store.delete(block.hash)
+    store.put(block)
+    await tx.done
+  }
+
+  async queryBlock(hash: `0x${string}`): Promise<BlockEntry | null> {
+    const db = await this.datasource
+    const block = await db.get('block', hash)
+    return block ?? null
+  }
+
+  async queryBlockByNumber(number: number): Promise<BlockEntry | null> {
+    const db = await this.datasource
+    const block = await db.getFromIndex('block', 'byNumber', number)
+    return block ?? null
+  }
+
+  async queryHighestBlock(): Promise<BlockEntry | null> {
+    const db = await this.datasource
+    const index = db.transaction('block').store.index('byNumber')
+    const cursor = await index.openCursor(null, 'prev')
+    return cursor?.value ?? null
+  }
+
+  async deleteBlock(hash: `0x${string}`): Promise<void> {
+    const db = await this.datasource
+    await db.delete('block', hash)
+  }
+
+  async blocksCount(): Promise<number> {
+    const db = await this.datasource
+    return db.count('block')
+  }
+
+  async saveStorage(blockHash: `0x${string}`, key: `0x${string}`, value: `0x${string}` | null): Promise<void> {
+    const db = await this.datasource
+    await db.put('keyValue', value, `${blockHash}-${key}`)
+  }
+
+  async queryStorage(blockHash: `0x${string}`, key: `0x${string}`): Promise<KeyValueEntry | null> {
+    const db = await this.datasource
+    const value = await db.get('keyValue', `${blockHash}-${key}`)
+    return value !== undefined ? { blockHash, key, value } : null
   }
 }
diff --git a/packages/db/src/db/browser.ts b/packages/db/src/db/browser.ts
diff --git a/packages/db/src/db/sql-wasm.ts b/packages/db/src/db/sql-wasm.ts
diff --git a/packages/web-test/src/App.tsx b/packages/web-test/src/App.tsx
@@ -15,11 +15,11 @@ import { ApiPromise } from '@polkadot/api'
 import { Buffer } from 'buffer'
 import { ChopsticksProvider, setStorage, setup } from '@acala-network/chopsticks-core'
 import { HexString } from '@polkadot/util/types'
+import { IdbDatabase } from '@acala-network/chopsticks-db/browser'
 import { createTestPairs } from '@polkadot/keyring'
 import { styled } from '@mui/system'
 import { useEffect, useState } from 'react'
 import type { SetupOptions } from '@acala-network/chopsticks-core'
-// import { SqljsDatabase } from '@acala-network/chopsticks-db/browser'
 
 window.Buffer = Buffer
 
@@ -109,8 +109,7 @@ function App() {
 			endpoint: config.endpoint,
 			block: config.block,
 			mockSignatureHost: true,
-			// disable cache, this makes it slower
-			// db: new SqljsDatabase('cache'),
+			db: new IdbDatabase('cache'),
 		})
 		globalThis.chain = chain
 

diff --git a/packages/web-test/tests/index.spec.ts b/packages/web-test/tests/index.spec.ts
@@ -1,3 +1,4 @@
+import { HexString } from '@polkadot/util/types'
 import { expect, test } from '@playwright/test'
 
 test.describe('index', () => {
@@ -20,6 +21,21 @@ test.describe('index', () => {
     await page.getByText(/build block/i).click()
     // wait for new block
     await expect(page.locator('#blocks-section')).toHaveText(/4000001/, { timeout: 200_000 })
+    // check indexedDB
+    const savedHash = await page.evaluate(async () => {
+      const db = window.indexedDB.open('cache', 1)
+      await new Promise((resolve) => {
+        db.onsuccess = resolve
+      })
+      const tx = db.result.transaction('block', 'readonly')
+      const store = tx.objectStore('block')
+      const block = store.index('byNumber').get(4000001)
+      await new Promise((resolve) => {
+        block.onsuccess = resolve
+      })
+      return block.result.hash
+    })
+    expect(savedHash).toBe('0x6b81a9a7fabbe32c1e685b944c8f1afd06be7e58ae48bb8d5ac50cc761d9bb77')
   })
 
   test('dry run extrinsic', async ({ page }) => {
@@ -30,4 +46,75 @@ test.describe('index', () => {
     await expect(page.getByText('Loading dry run result...')).toBeVisible()
     await expect(page.locator('#extrinsic-section')).toHaveText(/outcome/, { timeout: 200_000 })
   })
+
+  test('chain indexedDB works', async ({ page }) => {
+    test.setTimeout(6 * 60 * 1000) // 6 minutes timeout
+    // chain is ready
+    await expect(page.locator('#blocks-section')).toHaveText(/4000000/, { timeout: 60_000 })
+    await page.getByText(/build block/i).click()
+    // wait for new block
+    await expect(page.locator('#blocks-section')).toHaveText(/4000001/, { timeout: 200_000 })
+    await page.getByText(/build block/i).click()
+    // wait for new block
+    await expect(page.locator('#blocks-section')).toHaveText(/4000002/, { timeout: 20_000 })
+
+    // test db methods
+    const hightestBlock = await page.evaluate(() => globalThis.chain.db?.queryHighestBlock())
+    expect(hightestBlock).toEqual(
+      expect.objectContaining({
+        number: 4_000_002,
+        hash: '0xdd1d5206ce64d643e262f0bdc351147e2ba9e20846fdf78c9c5855ab6e2bc0ca',
+      }),
+    )
+    const blockByNumber = await page.evaluate(() => globalThis.chain.db?.queryBlockByNumber(4_000_001))
+    expect(blockByNumber).toEqual(
+      expect.objectContaining({
+        number: 4_000_001,
+        hash: '0x6b81a9a7fabbe32c1e685b944c8f1afd06be7e58ae48bb8d5ac50cc761d9bb77',
+      }),
+    )
+    const blocksCount = await page.evaluate(() => globalThis.chain.db?.blocksCount())
+    expect(blocksCount).toBe(2)
+    await page.evaluate(
+      (hightestBlock) => globalThis.chain.db?.deleteBlock(hightestBlock?.hash as HexString),
+      hightestBlock,
+    )
+    // run test again after deleting hightest block
+    {
+      const blocksCount = await page.evaluate(() => globalThis.chain.db?.blocksCount())
+      expect(blocksCount).toBe(1)
+      const hightestBlock = await page.evaluate(() => globalThis.chain.db?.queryHighestBlock())
+      expect(hightestBlock).toEqual(
+        expect.objectContaining({
+          number: 4_000_001,
+          hash: '0x6b81a9a7fabbe32c1e685b944c8f1afd06be7e58ae48bb8d5ac50cc761d9bb77',
+        }),
+      )
+    }
+
+    // test storage
+    {
+      const storage = await page.evaluate(() => globalThis.chain.db?.queryStorage('0xaa', '0x01'))
+      expect(storage).toBeNull()
+    }
+    {
+      await page.evaluate(() => globalThis.chain.db?.saveStorage('0xaa', '0x01', null))
+      const storage = await page.evaluate(() => globalThis.chain.db?.queryStorage('0xaa', '0x01'))
+      expect(storage).toEqual({ blockHash: '0xaa', key: '0x01', value: null })
+    }
+    {
+      await page.evaluate(() => globalThis.chain.db?.saveStorage('0xaa', '0x01', '0x01'))
+      const storage = await page.evaluate(() => globalThis.chain.db?.queryStorage('0xaa', '0x01'))
+      expect(storage).toEqual({ blockHash: '0xaa', key: '0x01', value: '0x01' })
+    }
+    {
+      await page.evaluate(() => globalThis.chain.db?.saveStorage('0xbb', '0x02', '0x02'))
+      const storage = await page.evaluate(() => globalThis.chain.db?.queryStorage('0xbb', '0x02'))
+      expect(storage).toEqual({ blockHash: '0xbb', key: '0x02', value: '0x02' })
+    }
+    {
+      const storage = await page.evaluate(() => globalThis.chain.db?.queryStorage('0xbb', '0x01'))
+      expect(storage).toBeNull()
+    }
+  })
 })