fix deleted key

.github/workflows/ci.yml

@@ -21,6 +21,7 @@ jobs:
           ~/.cargo/registry/cache/
           ~/.cargo/git/db/
           executor/target/
+          e2e-tests-db.sqlite
         key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
     - name: Install toolchain
       uses: dtolnay/rust-toolchain@nightly

packages/core/src/blockchain/storage-layer.ts

@@ -107,7 +107,7 @@ export class RemoteStorageLayer implements StorageLayerProvider {
 }
 
 export class StorageLayer implements StorageLayerProvider {
-  readonly #store: Record<string, StorageValue | Promise<StorageValue>> = {}
+  readonly #store: Map<string, StorageValue | Promise<StorageValue>> = new Map()
   readonly #keys: string[] = []
   readonly #deletedPrefix: string[] = []
   #parent?: StorageLayerProvider
@@ -134,8 +134,8 @@ export class StorageLayer implements StorageLayerProvider {
   }
 
   async get(key: string, cache: boolean): Promise<StorageValue | undefined> {
-    if (key in this.#store) {
-      return this.#store[key]
+    if (this.#store.has(key)) {
+      return this.#store.get(key)
     }
 
     if (this.#deletedPrefix.some((dp) => key.startsWith(dp))) {
@@ -145,7 +145,7 @@ export class StorageLayer implements StorageLayerProvider {
     if (this.#parent) {
       const val = this.#parent.get(key, false)
       if (cache) {
-        this.#store[key] = val
+        this.#store.set(key, val)
       }
       return val
     }
@@ -156,24 +156,24 @@ export class StorageLayer implements StorageLayerProvider {
   set(key: string, value: StorageValue): void {
     switch (value) {
       case StorageValueKind.Deleted:
-        this.#store[key] = value
+        this.#store.set(key, StorageValueKind.Deleted)
         this.#removeKey(key)
         break
       case StorageValueKind.DeletedPrefix:
         this.#deletedPrefix.push(key)
         for (const k of this.#keys) {
           if (k.startsWith(key)) {
-            this.#store[k] = StorageValueKind.Deleted
+            this.#store.set(k, StorageValueKind.Deleted)
             this.#removeKey(k)
           }
         }
         break
       case undefined:
-        delete this.#store[key]
+        this.#store.delete(key)
         this.#removeKey(key)
         break
       default:
-        this.#store[key] = value
+        this.#store.set(key, value)
         this.#addKey(key)
         break
     }
@@ -195,9 +195,8 @@ export class StorageLayer implements StorageLayerProvider {
       into.set(deletedPrefix, StorageValueKind.DeletedPrefix)
     }
 
-    for (const key of this.#keys) {
-      const value = await this.#store[key]
-      into.set(key, value)
+    for (const [key, value] of this.#store) {
+      into.set(key, await value)
     }
 
     return newParent
@@ -213,6 +212,9 @@ export class StorageLayer implements StorageLayerProvider {
     if (!this.#deletedPrefix.some((dp) => startKey.startsWith(dp))) {
       const remote = (await this.#parent?.getKeysPaged(prefix, pageSize, startKey)) ?? []
       for (const key of remote) {
+        if (this.#store.get(key) === StorageValueKind.Deleted) {
+          continue
+        }
         if (this.#deletedPrefix.some((dp) => key.startsWith(dp))) {
           continue
         }
@@ -240,7 +242,7 @@ export class StorageLayer implements StorageLayerProvider {
    * Merge the storage layer into the given object, can be used to get sotrage diff.
    */
   async mergeInto(into: Record<string, string | null>) {
-    for (const [key, maybeValue] of Object.entries(this.#store)) {
+    for (const [key, maybeValue] of this.#store) {
       const value = await maybeValue
       if (value === StorageValueKind.Deleted) {
         into[key] = null

packages/e2e/src/decoder.test.ts

@@ -9,11 +9,10 @@ const TOKENS_ACCOUNTS =
   '0x99971b5749ac43e0235e41b0d37869188ee7418a6531173d60d1f6a82d8f4d51de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01a12dfa1fa4ab9a0000'
 
 describe('decoder', async () => {
-  const acala = await networks.acala()
-  const { chain } = acala
+  const { chain, teardown } = await networks.acala()
 
   afterAll(async () => {
-    await acala.teardown()
+    await teardown()
   })
 
   it('decode keys', async () => {
@@ -41,7 +40,7 @@ describe('decoder', async () => {
     const data = { data: { free: 10000000000 } }
     const value = meta.registry.createType('AccountInfo', data)
     expect(decodeKeyValue(meta, chain.head, SYSTEM_ACCOUNT, value.toHex())).toMatchSnapshot()
-
+    await new Promise((resolve) => setTimeout(resolve, 1000))
     await teardown()
   })
 })

packages/e2e/src/helper.ts

@@ -1,5 +1,5 @@
 import { ApiPromise, WsProvider } from '@polkadot/api'
-import { Codec } from '@polkadot/types/types'
+import { Codec, RegisteredTypes } from '@polkadot/types/types'
 import { HexString } from '@polkadot/util/types'
 import { beforeAll, beforeEach, expect, vi } from 'vitest'
 
@@ -14,6 +14,7 @@ import {
   SetTimestamp,
   SetValidationData,
 } from '@acala-network/chopsticks-core/blockchain/inherent'
+import { SqliteDatabase } from '@acala-network/chopsticks-db'
 import { StorageValues } from '@acala-network/chopsticks-core/utils/set-storage'
 import { createServer } from '@acala-network/chopsticks/server'
 import { defer } from '@acala-network/chopsticks-core/utils'
@@ -28,6 +29,8 @@ export type SetupOption = {
   mockSignatureHost?: boolean
   allowUnresolvedImports?: boolean
   genesis?: string
+  registeredTypes?: RegisteredTypes
+  runtimeLogLevel?: number
 }
 
 export const env = {
@@ -52,6 +55,8 @@ export const setupAll = async ({
   mockSignatureHost,
   allowUnresolvedImports,
   genesis,
+  registeredTypes = {},
+  runtimeLogLevel,
 }: SetupOption) => {
   const api = new Api(genesis ? await genesisFromUrl(genesis) : new WsProvider(endpoint), {
     SetEvmOrigin: { payload: {}, extrinsic: {} },
@@ -88,12 +93,14 @@ export const setupAll = async ({
         },
         mockSignatureHost,
         allowUnresolvedImports,
-        registeredTypes: {},
+        registeredTypes,
+        runtimeLogLevel,
+        db: !process.env.RUN_TESTS_WITHOUT_DB ? new SqliteDatabase('e2e-tests-db.sqlite') : undefined,
       })
 
       const { port, close } = await createServer(handler({ chain }))
 
-      const ws = new WsProvider(`ws://localhost:${port}`)
+      const ws = new WsProvider(`ws://localhost:${port}`, undefined, undefined, 200_000)
       const apiPromise = await ApiPromise.create({
         provider: ws,
         signedExtensions: {
@@ -174,8 +181,10 @@ export const dev = {
 export const mockCallback = () => {
   let next = defer()
   const callback = vi.fn((...args) => {
-    next.resolve(args)
-    next = defer()
+    delay(100).then(() => {
+      next.resolve(args)
+      next = defer()
+    })
   })
 
   return {

packages/e2e/src/storage-migrate.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from 'vitest'
+
+import { api, dev, setupApi } from './helper'
+
+setupApi({
+  endpoint: 'wss://kusama-archive.mangata.online',
+  blockHash: '0xea25e5e478f33cf70eebcd4a8b94b8dde361537eaa7a8b53c58a03026a4ebac0',
+  mockSignatureHost: true,
+  runtimeLogLevel: 5,
+  registeredTypes: {
+    types: {
+      ShufflingSeed: {
+        seed: 'H256',
+        proof: 'H512',
+      },
+      Header: {
+        parentHash: 'Hash',
+        number: 'Compact<BlockNumber>',
+        stateRoot: 'Hash',
+        extrinsicsRoot: 'Hash',
+        digest: 'Digest',
+        seed: 'ShufflingSeed',
+        count: 'BlockNumber',
+      },
+    },
+  },
+})
+
+describe.runIf(process.env.CI)('storage-migrate', async () => {
+  it(
+    'no empty keys',
+    async () => {
+      await dev.newBlock()
+      const metadatas = await api.query.assetRegistry.metadata.entries()
+      expect(metadatas.some(([_, v]) => v.isEmpty)).toBeFalsy()
+    },
+    { timeout: 300_000 },
+  )
+})

vitest.config.mts

@@ -4,6 +4,8 @@ import tsconfigPaths from 'vite-tsconfig-paths'
 
 export default defineConfig({
 	test: {
+		minThreads: process.env.CI ? 1 : undefined /* use defaults */,
+		maxThreads: process.env.CI ? 4 : undefined /* use defaults */,
 		hookTimeout: 30000,
 		testTimeout: 120000,
 		include: ['packages/**/*.test.ts'],