From ceee9a2701ef14b75d19ecc7c359e5ad0af05e10 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 27 Oct 2023 12:35:50 +0200 Subject: [PATCH 1/4] fix deleted key --- .github/workflows/ci.yml | 1 + packages/core/src/blockchain/storage-layer.ts | 10 +++-- packages/e2e/src/decoder.test.ts | 7 ++-- packages/e2e/src/helper.ts | 19 ++++++--- packages/e2e/src/storage-migrate.test.ts | 39 +++++++++++++++++++ 5 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 packages/e2e/src/storage-migrate.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ec2b3be..7c9b02b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ executor/target/ + e2e-tests-db.sqlite key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Install toolchain uses: dtolnay/rust-toolchain@nightly diff --git a/packages/core/src/blockchain/storage-layer.ts b/packages/core/src/blockchain/storage-layer.ts index fcce5268..1df32859 100644 --- a/packages/core/src/blockchain/storage-layer.ts +++ b/packages/core/src/blockchain/storage-layer.ts @@ -156,7 +156,7 @@ export class StorageLayer implements StorageLayerProvider { set(key: string, value: StorageValue): void { switch (value) { case StorageValueKind.Deleted: - this.#store[key] = value + this.#store[key] = StorageValueKind.Deleted this.#removeKey(key) break case StorageValueKind.DeletedPrefix: @@ -195,9 +195,8 @@ export class StorageLayer implements StorageLayerProvider { into.set(deletedPrefix, StorageValueKind.DeletedPrefix) } - for (const key of this.#keys) { - const value = await this.#store[key] - into.set(key, value) + for (const [key, value] of Object.entries(this.#store)) { + into.set(key, await value) } return newParent @@ -216,6 +215,9 @@ export class StorageLayer implements StorageLayerProvider { if (this.#deletedPrefix.some((dp) => key.startsWith(dp))) { continue } + if (this.#store[key] === StorageValueKind.Deleted) { + continue + } this.#addKey(key) } } diff --git a/packages/e2e/src/decoder.test.ts b/packages/e2e/src/decoder.test.ts index e18d3c28..5db0a033 100644 --- a/packages/e2e/src/decoder.test.ts +++ b/packages/e2e/src/decoder.test.ts @@ -9,11 +9,10 @@ const TOKENS_ACCOUNTS = '0x99971b5749ac43e0235e41b0d37869188ee7418a6531173d60d1f6a82d8f4d51de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01a12dfa1fa4ab9a0000' describe('decoder', async () => { - const acala = await networks.acala() - const { chain } = acala + const { chain, teardown } = await networks.acala() afterAll(async () => { - await acala.teardown() + await teardown() }) it('decode keys', async () => { @@ -41,7 +40,7 @@ describe('decoder', async () => { const data = { data: { free: 10000000000 } } const value = meta.registry.createType('AccountInfo', data) expect(decodeKeyValue(meta, chain.head, SYSTEM_ACCOUNT, value.toHex())).toMatchSnapshot() - + await new Promise((resolve) => setTimeout(resolve, 1000)) await teardown() }) }) diff --git a/packages/e2e/src/helper.ts b/packages/e2e/src/helper.ts index 8e548206..fc077377 100644 --- a/packages/e2e/src/helper.ts +++ b/packages/e2e/src/helper.ts @@ -1,5 +1,5 @@ import { ApiPromise, WsProvider } from '@polkadot/api' -import { Codec } from '@polkadot/types/types' +import { Codec, RegisteredTypes } from '@polkadot/types/types' import { HexString } from '@polkadot/util/types' import { beforeAll, beforeEach, expect, vi } from 'vitest' @@ -14,6 +14,7 @@ import { SetTimestamp, SetValidationData, } from '@acala-network/chopsticks-core/blockchain/inherent' +import { SqliteDatabase } from '@acala-network/chopsticks-db' import { StorageValues } from '@acala-network/chopsticks-core/utils/set-storage' import { createServer } from '@acala-network/chopsticks/server' import { defer } from '@acala-network/chopsticks-core/utils' @@ -28,6 +29,8 @@ export type SetupOption = { mockSignatureHost?: boolean allowUnresolvedImports?: boolean genesis?: string + registeredTypes?: RegisteredTypes + runtimeLogLevel?: number } export const env = { @@ -52,6 +55,8 @@ export const setupAll = async ({ mockSignatureHost, allowUnresolvedImports, genesis, + registeredTypes = {}, + runtimeLogLevel, }: SetupOption) => { const api = new Api(genesis ? await genesisFromUrl(genesis) : new WsProvider(endpoint), { SetEvmOrigin: { payload: {}, extrinsic: {} }, @@ -88,12 +93,14 @@ export const setupAll = async ({ }, mockSignatureHost, allowUnresolvedImports, - registeredTypes: {}, + registeredTypes, + runtimeLogLevel, + db: !process.env.RUN_TESTS_WITHOUT_DB ? new SqliteDatabase('e2e-tests-db.sqlite') : undefined, }) const { port, close } = await createServer(handler({ chain })) - const ws = new WsProvider(`ws://localhost:${port}`) + const ws = new WsProvider(`ws://localhost:${port}`, undefined, undefined, 200_000) const apiPromise = await ApiPromise.create({ provider: ws, signedExtensions: { @@ -174,8 +181,10 @@ export const dev = { export const mockCallback = () => { let next = defer() const callback = vi.fn((...args) => { - next.resolve(args) - next = defer() + delay(100).then(() => { + next.resolve(args) + next = defer() + }) }) return { diff --git a/packages/e2e/src/storage-migrate.test.ts b/packages/e2e/src/storage-migrate.test.ts new file mode 100644 index 00000000..d57396e2 --- /dev/null +++ b/packages/e2e/src/storage-migrate.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' + +import { api, dev, setupApi } from './helper' + +setupApi({ + endpoint: 'wss://kusama-archive.mangata.online', + blockHash: '0xea25e5e478f33cf70eebcd4a8b94b8dde361537eaa7a8b53c58a03026a4ebac0', + mockSignatureHost: true, + runtimeLogLevel: 5, + registeredTypes: { + types: { + ShufflingSeed: { + seed: 'H256', + proof: 'H512', + }, + Header: { + parentHash: 'Hash', + number: 'Compact', + stateRoot: 'Hash', + extrinsicsRoot: 'Hash', + digest: 'Digest', + seed: 'ShufflingSeed', + count: 'BlockNumber', + }, + }, + }, +}) + +describe.runIf(process.env.CI)('storage-migrate', async () => { + it( + 'no empty keys', + async () => { + await dev.newBlock() + const metadatas = await api.query.assetRegistry.metadata.entries() + expect(metadatas.some(([_, v]) => v.isEmpty)).toBeFalsy() + }, + { timeout: 200_000 }, + ) +}) From ec249f8ff706b29f69e7c710e607048dec341217 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 27 Oct 2023 13:11:43 +0200 Subject: [PATCH 2/4] Increase timeout --- packages/e2e/src/storage-migrate.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/e2e/src/storage-migrate.test.ts b/packages/e2e/src/storage-migrate.test.ts index d57396e2..111833a6 100644 --- a/packages/e2e/src/storage-migrate.test.ts +++ b/packages/e2e/src/storage-migrate.test.ts @@ -34,6 +34,6 @@ describe.runIf(process.env.CI)('storage-migrate', async () => { const metadatas = await api.query.assetRegistry.metadata.entries() expect(metadatas.some(([_, v]) => v.isEmpty)).toBeFalsy() }, - { timeout: 200_000 }, + { timeout: 300_000 }, ) }) From 00e46371f711883e40334bc445cd54efd4bdb53b Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 27 Oct 2023 16:40:58 +0200 Subject: [PATCH 3/4] use Map for better performance --- packages/core/src/blockchain/storage-layer.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/core/src/blockchain/storage-layer.ts b/packages/core/src/blockchain/storage-layer.ts index 1df32859..5c75b553 100644 --- a/packages/core/src/blockchain/storage-layer.ts +++ b/packages/core/src/blockchain/storage-layer.ts @@ -107,7 +107,7 @@ export class RemoteStorageLayer implements StorageLayerProvider { } export class StorageLayer implements StorageLayerProvider { - readonly #store: Record> = {} + readonly #store: Map> = new Map() readonly #keys: string[] = [] readonly #deletedPrefix: string[] = [] #parent?: StorageLayerProvider @@ -134,8 +134,8 @@ export class StorageLayer implements StorageLayerProvider { } async get(key: string, cache: boolean): Promise { - if (key in this.#store) { - return this.#store[key] + if (this.#store.has(key)) { + return this.#store.get(key) } if (this.#deletedPrefix.some((dp) => key.startsWith(dp))) { @@ -145,7 +145,7 @@ export class StorageLayer implements StorageLayerProvider { if (this.#parent) { const val = this.#parent.get(key, false) if (cache) { - this.#store[key] = val + this.#store.set(key, val) } return val } @@ -156,24 +156,24 @@ export class StorageLayer implements StorageLayerProvider { set(key: string, value: StorageValue): void { switch (value) { case StorageValueKind.Deleted: - this.#store[key] = StorageValueKind.Deleted + this.#store.set(key, StorageValueKind.Deleted) this.#removeKey(key) break case StorageValueKind.DeletedPrefix: this.#deletedPrefix.push(key) for (const k of this.#keys) { if (k.startsWith(key)) { - this.#store[k] = StorageValueKind.Deleted + this.#store.set(k, StorageValueKind.Deleted) this.#removeKey(k) } } break case undefined: - delete this.#store[key] + this.#store.delete(key) this.#removeKey(key) break default: - this.#store[key] = value + this.#store.set(key, value) this.#addKey(key) break } @@ -195,7 +195,7 @@ export class StorageLayer implements StorageLayerProvider { into.set(deletedPrefix, StorageValueKind.DeletedPrefix) } - for (const [key, value] of Object.entries(this.#store)) { + for (const [key, value] of this.#store) { into.set(key, await value) } @@ -212,10 +212,10 @@ export class StorageLayer implements StorageLayerProvider { if (!this.#deletedPrefix.some((dp) => startKey.startsWith(dp))) { const remote = (await this.#parent?.getKeysPaged(prefix, pageSize, startKey)) ?? [] for (const key of remote) { - if (this.#deletedPrefix.some((dp) => key.startsWith(dp))) { + if (this.#store.get(key) === StorageValueKind.Deleted) { continue } - if (this.#store[key] === StorageValueKind.Deleted) { + if (this.#deletedPrefix.some((dp) => key.startsWith(dp))) { continue } this.#addKey(key) @@ -242,7 +242,7 @@ export class StorageLayer implements StorageLayerProvider { * Merge the storage layer into the given object, can be used to get sotrage diff. */ async mergeInto(into: Record) { - for (const [key, maybeValue] of Object.entries(this.#store)) { + for (const [key, maybeValue] of this.#store) { const value = await maybeValue if (value === StorageValueKind.Deleted) { into[key] = null From c7c74a68538e6b3cd9149c45468d85cfbf5a5f7d Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 27 Oct 2023 16:55:09 +0200 Subject: [PATCH 4/4] enable threads --- vitest.config.mts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vitest.config.mts b/vitest.config.mts index d6d4ebde..f4c9ac56 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -4,6 +4,8 @@ import tsconfigPaths from 'vite-tsconfig-paths' export default defineConfig({ test: { + minThreads: process.env.CI ? 1 : undefined /* use defaults */, + maxThreads: process.env.CI ? 4 : undefined /* use defaults */, hookTimeout: 30000, testTimeout: 120000, include: ['packages/**/*.test.ts'],