diff --git a/.eslintrc b/.eslintrc index 8a242c65..ad871c7f 100644 --- a/.eslintrc +++ b/.eslintrc @@ -130,7 +130,8 @@ "@typescript-eslint/no-unnecessary-condition": [ "off" ], - "nodejs/declare": "off" + "nodejs/declare": "off", + "unicorn/prefer-event-target": "off" }, "overrides": [ diff --git a/packages/api/src/graphql/modules/BlockStorageResolver.ts b/packages/api/src/graphql/modules/BlockStorageResolver.ts index eb3e0921..d5f2ef94 100644 --- a/packages/api/src/graphql/modules/BlockStorageResolver.ts +++ b/packages/api/src/graphql/modules/BlockStorageResolver.ts @@ -1,12 +1,6 @@ /* eslint-disable new-cap */ import { inject, injectable } from "tsyringe"; -import { - Arg, - Field, - ObjectType, - Query, - Resolver, -} from "type-graphql"; +import { Arg, Field, ObjectType, Query, Resolver } from "type-graphql"; import { IsBoolean } from "class-validator"; import { BlockStorage, @@ -53,10 +47,15 @@ export class ComputedBlockTransactionModel { @ObjectType() export class ComputedBlockModel { - public static fromServiceLayerModel({ txs, proof }: ComputedBlock) { + public static fromServiceLayerModel({ + txs, + proof, + }: ComputedBlock): ComputedBlockModel { return new ComputedBlockModel( txs.map((tx) => ComputedBlockTransactionModel.fromServiceLayerModel(tx)), - JSON.stringify(proof.toJSON()) + proof.proof === "mock-proof" + ? "mock-proof" + : JSON.stringify(proof.toJSON()) ); } @@ -74,7 +73,6 @@ export class ComputedBlockModel { @graphqlModule() export class BlockStorageResolver extends GraphqlModule { - // TODO seperate these two block interfaces public constructor( @inject("BlockStorage") private readonly blockStorage: BlockStorage & HistoricalBlockStorage @@ -83,12 +81,12 @@ export class BlockStorageResolver extends GraphqlModule { } @Query(() => ComputedBlockModel, { nullable: true }) - public async block( + public async settlements( @Arg("height", () => Number, { nullable: true }) height: number | undefined ) { const blockHeight = - height ?? (await this.blockStorage.getCurrentBlockHeight()); + height ?? (await this.blockStorage.getCurrentBlockHeight()) - 1; const block = await this.blockStorage.getBlockAt(blockHeight); diff --git a/packages/api/src/graphql/modules/NodeStatusResolver.ts b/packages/api/src/graphql/modules/NodeStatusResolver.ts index 647eb90f..6fb65664 100644 --- a/packages/api/src/graphql/modules/NodeStatusResolver.ts +++ b/packages/api/src/graphql/modules/NodeStatusResolver.ts @@ -10,7 +10,8 @@ export class NodeStatusObject { return new NodeStatusObject( status.uptime, status.uptimeHumanReadable, - status.height + status.height, + status.settlements ); } @@ -20,17 +21,22 @@ export class NodeStatusObject { @Field() public height: number; + @Field() + public settlements: number; + @Field() public uptimeHumanReadable: string; public constructor( uptime: number, uptimeHumanReadable: string, - height: number + height: number, + settlements: number ) { this.uptime = uptime; this.uptimeHumanReadable = uptimeHumanReadable; this.height = height; + this.settlements = settlements; } } diff --git a/packages/api/src/graphql/modules/UnprovenBlockResolver.ts b/packages/api/src/graphql/modules/UnprovenBlockResolver.ts new file mode 100644 index 00000000..f100c84b --- /dev/null +++ b/packages/api/src/graphql/modules/UnprovenBlockResolver.ts @@ -0,0 +1,65 @@ +import { inject } from "tsyringe"; +import { + HistoricalUnprovenBlockStorage, + UnprovenBlock, + UnprovenBlockStorage, +} from "@proto-kit/sequencer"; +import { Arg, Field, ObjectType, Query } from "type-graphql"; + +import { GraphqlModule, graphqlModule } from "../GraphqlModule"; + +import { ComputedBlockTransactionModel } from "./BlockStorageResolver"; + +@ObjectType() +export class UnprovenBlockModel { + public static fromServiceLayerModel(unprovenBlock: UnprovenBlock) { + return new UnprovenBlockModel( + Number(unprovenBlock.networkState.block.height.toBigInt()), + unprovenBlock.transactions.map((tx) => + ComputedBlockTransactionModel.fromServiceLayerModel({ + tx: tx.tx, + status: tx.status.toBoolean(), + statusMessage: tx.statusMessage, + }) + ) + ); + } + + @Field() + height: number; + + @Field(() => [ComputedBlockTransactionModel]) + txs: ComputedBlockTransactionModel[]; + + private constructor(height: number, txs: ComputedBlockTransactionModel[]) { + this.height = height; + this.txs = txs; + } +} + +@graphqlModule() +export class UnprovenBlockResolver extends GraphqlModule { + public constructor( + @inject("UnprovenBlockStorage") + private readonly blockStorage: HistoricalUnprovenBlockStorage & + UnprovenBlockStorage + ) { + super(); + } + + @Query(() => UnprovenBlockModel, { nullable: true }) + public async block( + @Arg("height", () => Number, { nullable: true }) + height: number | undefined + ) { + const blockHeight = + height ?? (await this.blockStorage.getCurrentBlockHeight()) - 1; + + const block = await this.blockStorage.getBlockAt(blockHeight); + + if (block !== undefined) { + return UnprovenBlockModel.fromServiceLayerModel(block); + } + return undefined; + } +} diff --git a/packages/api/src/graphql/services/NodeStatusService.ts b/packages/api/src/graphql/services/NodeStatusService.ts index 550772e0..a5454264 100644 --- a/packages/api/src/graphql/services/NodeStatusService.ts +++ b/packages/api/src/graphql/services/NodeStatusService.ts @@ -1,11 +1,12 @@ import { inject, injectable } from "tsyringe"; -import { BlockStorage } from "@proto-kit/sequencer"; +import { BlockStorage, UnprovenBlockStorage } from "@proto-kit/sequencer"; import humanizeDuration from "humanize-duration"; export interface NodeStatus { uptime: number; uptimeHumanReadable: string; height: number; + settlements: number; } @injectable() @@ -13,19 +14,22 @@ export class NodeStatusService { private readonly startupTime = Date.now(); public constructor( + @inject("UnprovenBlockStorage") + private readonly unprovenBlockStorage: UnprovenBlockStorage, @inject("BlockStorage") private readonly blockStorage: BlockStorage - ) { - } + ) {} public async getNodeStatus(): Promise { const uptime = Date.now() - this.startupTime; const uptimeHumanReadable = humanizeDuration(uptime); - const height = await this.blockStorage.getCurrentBlockHeight(); + const height = await this.unprovenBlockStorage.getCurrentBlockHeight(); + const settlements = await this.blockStorage.getCurrentBlockHeight(); return { uptime, uptimeHumanReadable, height, + settlements, }; } } diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index b91505a4..68b84f23 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -1,6 +1,7 @@ export * from "./graphql/modules/QueryGraphqlModule"; export * from "./graphql/modules/MempoolResolver"; export * from "./graphql/modules/BlockStorageResolver"; +export * from "./graphql/modules/UnprovenBlockResolver"; export * from "./graphql/GraphqlModule"; export * from "./graphql/GraphqlServer"; export * from "./graphql/GraphqlSequencerModule"; diff --git a/packages/common/src/events/EventEmitter.ts b/packages/common/src/events/EventEmitter.ts new file mode 100644 index 00000000..3b061ca0 --- /dev/null +++ b/packages/common/src/events/EventEmitter.ts @@ -0,0 +1,68 @@ +import { EventsRecord } from "./EventEmittingComponent"; + +type ListenersHolder = { + [key in keyof Events]?: { + id: number; + listener: (...args: Events[key]) => void; + }[]; +}; + +export class EventEmitter { + private readonly listeners: ListenersHolder = {}; + + private counter = 0; + + // Fields used for offSelf() + private currentListenerId: number | undefined = undefined; + + private currentListenerEventName: keyof Events | undefined = undefined; + + public emit(event: keyof Events, ...parameters: Events[typeof event]) { + const listeners = this.listeners[event]; + if (listeners !== undefined) { + this.currentListenerEventName = event; + + listeners.forEach((listener) => { + this.currentListenerId = listener.id; + + listener.listener(...parameters); + + this.currentListenerId = undefined; + }); + this.currentListenerEventName = undefined; + } + } + + public on( + event: Key, + listener: (...args: Events[Key]) => void + ): number { + // eslint-disable-next-line no-multi-assign + const id = (this.counter += 1); + (this.listeners[event] ??= []).push({ + id, + listener, + }); + return id; + } + + // eslint-disable-next-line no-warning-comments + // TODO Improve to be thread-safe + public offSelf() { + if ( + this.currentListenerEventName !== undefined && + this.currentListenerId !== undefined + ) { + this.off(this.currentListenerEventName, this.currentListenerId); + } + } + + public off(event: Key, id: number) { + const listeners = this.listeners[event]; + if (listeners !== undefined) { + this.listeners[event] = listeners.filter( + (listener) => listener.id !== id + ); + } + } +} diff --git a/packages/common/src/events/EventEmittingComponent.ts b/packages/common/src/events/EventEmittingComponent.ts new file mode 100644 index 00000000..451084b5 --- /dev/null +++ b/packages/common/src/events/EventEmittingComponent.ts @@ -0,0 +1,11 @@ +import type { EventEmitter } from "./EventEmitter"; + +export type EventSignature = unknown[]; + +export interface EventsRecord { + [key: string]: EventSignature; +} + +export interface EventEmittingComponent { + events: EventEmitter; +} diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index 5d334acc..530c1001 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -9,3 +9,6 @@ export * from "./zkProgrammable/provableMethod"; export * from "./utils"; export * from "./dependencyFactory/DependencyFactory"; export * from "./log"; +export * from "./quickmaths"; +export * from "./events/EventEmittingComponent"; +export * from "./events/EventEmitter"; diff --git a/packages/common/src/quickmaths.ts b/packages/common/src/quickmaths.ts new file mode 100644 index 00000000..eda3f84f --- /dev/null +++ b/packages/common/src/quickmaths.ts @@ -0,0 +1,24 @@ +/** + * Computes the greatest common divisor of a and b + * @param a + * @param b + */ +export function gcd(a: number, b: number): number { + a = Math.abs(a); + b = Math.abs(b); + if (b > a) { + a = b; + b = a; + } + while (a > 0 && b > 0) { + if (b === 0) { + return a; + } + a %= b; + if (a === 0) { + return b; + } + b %= a; + } + return 1; +} diff --git a/packages/protocol/src/utils/merkletree/InMemoryMerkleTreeStorage.ts b/packages/protocol/src/utils/merkletree/InMemoryMerkleTreeStorage.ts index 1fa83ad7..390b87fc 100644 --- a/packages/protocol/src/utils/merkletree/InMemoryMerkleTreeStorage.ts +++ b/packages/protocol/src/utils/merkletree/InMemoryMerkleTreeStorage.ts @@ -1,7 +1,7 @@ import { MerkleTreeStore } from "./MerkleTreeStore"; export class InMemoryMerkleTreeStorage implements MerkleTreeStore { - protected readonly nodes: { + protected nodes: { [key: number]: { [key: string]: bigint; }; diff --git a/packages/protocol/src/utils/merkletree/MerkleTreeStore.ts b/packages/protocol/src/utils/merkletree/MerkleTreeStore.ts index a03c4923..595c9065 100644 --- a/packages/protocol/src/utils/merkletree/MerkleTreeStore.ts +++ b/packages/protocol/src/utils/merkletree/MerkleTreeStore.ts @@ -1,13 +1,3 @@ -export interface AsyncMerkleTreeStore { - openTransaction: () => void; - - commit: () => void; - - setNodeAsync: (key: bigint, level: number, value: bigint) => Promise; - - getNodeAsync: (key: bigint, level: number) => Promise; -} - export interface MerkleTreeStore { setNode: (key: bigint, level: number, value: bigint) => void; diff --git a/packages/sdk/src/appChain/TestingAppChain.ts b/packages/sdk/src/appChain/TestingAppChain.ts index 94e018d3..584903a3 100644 --- a/packages/sdk/src/appChain/TestingAppChain.ts +++ b/packages/sdk/src/appChain/TestingAppChain.ts @@ -18,6 +18,7 @@ import { BlockProducerModule, ManualBlockTrigger, LocalTaskQueue, + UnprovenProducerModule } from "@proto-kit/sequencer"; import { PrivateKey } from "o1js"; import { StateServiceQueryModule } from "../query/StateServiceQueryModule"; @@ -55,6 +56,7 @@ export class TestingAppChain< LocalTaskWorkerModule, BaseLayer: NoopBaseLayer, BlockProducerModule, + UnprovenProducerModule, BlockTrigger: ManualBlockTrigger, TaskQueue: LocalTaskQueue, }, @@ -65,6 +67,7 @@ export class TestingAppChain< BlockProducerModule: {}, LocalTaskWorkerModule: {}, BaseLayer: {}, + UnprovenProducerModule: {}, TaskQueue: { simulatedDuration: 0, @@ -72,7 +75,7 @@ export class TestingAppChain< }, }); - return new TestingAppChain({ + const appchain = new TestingAppChain({ runtime, sequencer, @@ -90,14 +93,36 @@ export class TestingAppChain< Signer: InMemorySigner, TransactionSender: InMemoryTransactionSender, QueryTransportModule: StateServiceQueryModule, + } + }); + + appchain.configure({ + Runtime: definition.config, + + Sequencer: { + BlockTrigger: {}, + Mempool: {}, + BlockProducerModule: {}, + LocalTaskWorkerModule: {}, + BaseLayer: {}, + UnprovenProducerModule: {}, + + TaskQueue: { + simulatedDuration: 0, + }, }, - config: { - Signer: {}, - TransactionSender: {}, - QueryTransportModule: {}, + Protocol: { + BlockProver: {}, + StateTransitionProver: {}, }, + + Signer: {}, + TransactionSender: {}, + QueryTransportModule: {}, }); + + return appchain; } public setSigner(signer: PrivateKey) { @@ -126,6 +151,6 @@ export class TestingAppChain< ManualBlockTrigger ); - return await blockTrigger.produceBlock(); + return await blockTrigger.produceUnproven(true); } } diff --git a/packages/sdk/test/graphql/graphql.ts b/packages/sdk/test/graphql/graphql.ts index c1cae873..f624765f 100644 --- a/packages/sdk/test/graphql/graphql.ts +++ b/packages/sdk/test/graphql/graphql.ts @@ -14,11 +14,17 @@ import { StateMap, VanillaProtocol, } from "@proto-kit/protocol"; -import { Presets, log } from "@proto-kit/common"; +import { Presets, log, sleep } from "@proto-kit/common"; import { - AsyncStateService, BlockProducerModule, LocalTaskQueue, LocalTaskWorkerModule, NoopBaseLayer, + AsyncStateService, + BlockProducerModule, + LocalTaskQueue, + LocalTaskWorkerModule, + NoopBaseLayer, + PendingTransaction, PrivateMempool, - Sequencer, TimedBlockTrigger, + Sequencer, + TimedBlockTrigger, UnsignedTransaction } from "@proto-kit/sequencer"; import { @@ -27,7 +33,7 @@ import { GraphqlServer, MempoolResolver, NodeStatusResolver, - QueryGraphqlModule + QueryGraphqlModule, UnprovenBlockResolver } from "@proto-kit/api"; import { AppChain } from "../../src/appChain/AppChain"; @@ -35,6 +41,9 @@ import { StateServiceQueryModule } from "../../src/query/StateServiceQueryModule import { InMemorySigner } from "../../src/transaction/InMemorySigner"; import { InMemoryTransactionSender } from "../../src/transaction/InMemoryTransactionSender"; import { container } from "tsyringe"; +import { + UnprovenProducerModule +} from "@proto-kit/sequencer/dist/protocol/production/unproven/UnprovenProducerModule"; log.setLevel(log.levels.INFO); @@ -65,7 +74,7 @@ export class Balances extends RuntimeModule { UInt64 ); - @state() public totalSupply = State.from(UInt64); + @state() public totalSupply = State.from(UInt64); @runtimeMethod() public getBalance(address: PublicKey): Option { @@ -73,13 +82,19 @@ export class Balances extends RuntimeModule { } @runtimeMethod() - public setBalance(address: PublicKey, balance: UInt64) { - this.balances.set(address, balance); + public addBalance(address: PublicKey, balance: UInt64) { + const totalSupply = this.totalSupply.get() + this.totalSupply.set(totalSupply.orElse(UInt64.zero).add(balance)); + + const previous = this.balances.get(address) + this.balances.set(address, previous.orElse(UInt64.zero).add(balance)); } } export async function startServer() { + // log.setLevel("DEBUG") + const appChain = AppChain.from({ runtime: Runtime.from({ modules: { @@ -103,6 +118,7 @@ export async function startServer() { LocalTaskWorkerModule, BaseLayer: NoopBaseLayer, BlockProducerModule, + UnprovenProducerModule, BlockTrigger: TimedBlockTrigger, TaskQueue: LocalTaskQueue, @@ -111,6 +127,7 @@ export async function startServer() { MempoolResolver, QueryGraphqlModule, BlockStorageResolver, + UnprovenBlockResolver, NodeStatusResolver, }, @@ -119,6 +136,7 @@ export async function startServer() { QueryGraphqlModule: {}, BlockStorageResolver: {}, NodeStatusResolver: {}, + UnprovenBlockResolver: {} }, }), }, @@ -153,7 +171,8 @@ export async function startServer() { QueryGraphqlModule: {}, MempoolResolver: {}, BlockStorageResolver: {}, - NodeStatusResolver: {} + NodeStatusResolver: {}, + UnprovenBlockResolver: {} }, Mempool: {}, @@ -161,9 +180,11 @@ export async function startServer() { LocalTaskWorkerModule: {}, BaseLayer: {}, TaskQueue: {}, + UnprovenProducerModule: {}, BlockTrigger: { - blocktime: 5000 + blockInterval: 15000, + settlementInterval: 30000, }, }, @@ -176,7 +197,6 @@ export async function startServer() { }); await appChain.start(container.createChildContainer()); - const pk = PublicKey.fromBase58( "B62qmETai5Y8vvrmWSU8F4NX7pTyPqYLMhc1pgX3wD8dGc2wbCWUcqP" ); @@ -184,14 +204,36 @@ export async function startServer() { const balances = appChain.runtime.resolve("Balances"); + const priv = PrivateKey.fromBase58( + "EKFEMDTUV2VJwcGmCwNKde3iE1cbu7MHhzBqTmBtGAd6PdsLTifY" + ); + + const tx = appChain.transaction(priv.toPublicKey(), () => { + balances.addBalance(priv.toPublicKey(), UInt64.from(1000)) + }) + appChain.resolve("Signer").config.signer = priv + await tx.sign(); + await tx.send(); + // console.log((tx.transaction as PendingTransaction).toJSON()) + + const tx2 = appChain.transaction(priv.toPublicKey(), () => { + balances.addBalance(priv.toPublicKey(), UInt64.from(1000)) + }, {nonce: 0}) + await tx2.sign(); + await tx2.send(); + console.log("Path:", balances.balances.getPath(pk).toString()); - const asyncState = - appChain.sequencer.dependencyContainer.resolve( - "AsyncStateService" - ); - await asyncState.setAsync(balances.balances.getPath(pk), [Field(100)]); - await asyncState.setAsync(balances.totalSupply.path!, [Field(10_000)]); + // const asyncState = + // appChain.sequencer.dependencyContainer.resolve( + // "AsyncStateService" + // ); + // await asyncState.setAsync(balances.balances.getPath(pk), [Field(100)]); + // await asyncState.setAsync(balances.totalSupply.path!, [Field(10_000)]); + + // appChain.query.runtime.Balances.totalSupply + + // await sleep(30000); return appChain } diff --git a/packages/sequencer/src/index.ts b/packages/sequencer/src/index.ts index f15d8f1a..3914e700 100644 --- a/packages/sequencer/src/index.ts +++ b/packages/sequencer/src/index.ts @@ -19,12 +19,6 @@ export * from "./worker/worker/FlowTaskWorker"; export * from "./worker/worker/LocalTaskWorkerModule"; export * from "./protocol/baselayer/BaseLayer"; export * from "./protocol/baselayer/NoopBaseLayer"; -export * from "./protocol/production/execution/CachedStateService"; -export * from "./protocol/production/execution/DummyStateService"; -export * from "./protocol/production/execution/MerkleStoreWitnessProvider"; -export * from "./protocol/production/state/AsyncStateService"; -export * from "./protocol/production/tasks/providers/PreFilledStateService"; -export * from "./protocol/production/tasks/providers/PreFilledWitnessProvider"; export * from "./protocol/production/tasks/BlockProvingTask"; export * from "./protocol/production/tasks/CompileRegistry"; export * from "./protocol/production/tasks/RuntimeProvingTask"; @@ -34,14 +28,29 @@ export * from "./protocol/production/tasks/StateTransitionTaskParameters"; export * from "./protocol/production/trigger/BlockTrigger"; export * from "./protocol/production/trigger/ManualBlockTrigger"; export * from "./protocol/production/trigger/TimedBlockTrigger"; +export * from "./protocol/production/trigger/AutomaticBlockTrigger"; export * from "./protocol/production/BlockProducerModule"; export * from "./protocol/production/BlockTaskFlowService"; export * from "./protocol/production/TransactionTraceService"; +export * from "./protocol/production/unproven/RuntimeMethodExecution"; +export * from "./protocol/production/unproven/TransactionExecutionService"; +export * from "./protocol/production/unproven/UnprovenProducerModule"; export * from "./storage/model/Block"; export * from "./storage/repositories/BlockStorage"; +export * from "./storage/repositories/UnprovenBlockStorage"; export * from "./storage/MockStorageDependencyFactory"; export * from "./storage/Database"; export * from "./storage/StorageDependencyFactory"; export * from "./helpers/query/QueryTransportModule"; export * from "./helpers/query/QueryBuilderFactory"; export * from "./helpers/query/NetworkStateQuery"; +export * from "./state/prefilled/PreFilledStateService"; +export * from "./state/prefilled/PreFilledWitnessProvider"; +export * from "./state/async/AsyncMerkleTreeStore"; +export * from "./state/async/AsyncStateService"; +export * from "./state/merkle/CachedMerkleTreeStore"; +export * from "./state/merkle/SyncCachedMerkleTreeStore"; +export * from "./state/state/DummyStateService"; +export * from "./state/state/SyncCachedStateService"; +export * from "./state/state/CachedStateService"; +export * from "./state/MerkleStoreWitnessProvider"; diff --git a/packages/sequencer/src/mempool/Mempool.ts b/packages/sequencer/src/mempool/Mempool.ts index 44ee133a..6d8924d6 100644 --- a/packages/sequencer/src/mempool/Mempool.ts +++ b/packages/sequencer/src/mempool/Mempool.ts @@ -1,4 +1,5 @@ import type { Field } from "o1js"; +import { EventEmittingComponent, EventsRecord } from "@proto-kit/common"; import type { PendingTransaction } from "./PendingTransaction.js"; @@ -6,7 +7,12 @@ export interface MempoolCommitment { transactionsHash: Field; } -export interface Mempool { +export interface MempoolEvents extends EventsRecord { + transactionAdded: [PendingTransaction, MempoolCommitment]; + transactionsRemoved: [PendingTransaction[]] +} + +export interface Mempool extends EventEmittingComponent { /** * Add a transaction to the mempool * @returns The new commitment to the mempool diff --git a/packages/sequencer/src/mempool/private/PrivateMempool.ts b/packages/sequencer/src/mempool/private/PrivateMempool.ts index e68df88a..07a0cb7d 100644 --- a/packages/sequencer/src/mempool/private/PrivateMempool.ts +++ b/packages/sequencer/src/mempool/private/PrivateMempool.ts @@ -1,16 +1,19 @@ import { Field, Poseidon } from "o1js"; -import { noop } from "@proto-kit/common"; +import { EventEmitter, noop } from "@proto-kit/common"; -import type { Mempool, MempoolCommitment } from "../Mempool.js"; +import type { Mempool, MempoolCommitment, MempoolEvents } from "../Mempool.js"; import type { PendingTransaction } from "../PendingTransaction.js"; -import { SequencerModule } from "../../sequencer/builder/SequencerModule"; +import { sequencerModule, SequencerModule } from "../../sequencer/builder/SequencerModule"; import { TransactionValidator } from "../verification/TransactionValidator"; +@sequencerModule() export class PrivateMempool extends SequencerModule implements Mempool { public commitment: Field; private queue: PendingTransaction[] = []; + public events = new EventEmitter(); + public constructor( private readonly transactionValidator: TransactionValidator ) { @@ -26,6 +29,8 @@ export class PrivateMempool extends SequencerModule implements Mempool { // Figure out how to generalize this this.commitment = Poseidon.hash([this.commitment, tx.hash()]); + this.events.emit("transactionAdded", [tx, this.commitment]); + return { transactionsHash: this.commitment }; } throw new Error(`Valdiation of tx failed: ${error ?? "unknown error"}`); @@ -44,7 +49,12 @@ export class PrivateMempool extends SequencerModule implements Mempool { public removeTxs(txs: PendingTransaction[]): boolean { const { length } = this.queue; this.queue = this.queue.filter((tx) => !txs.includes(tx)); + + this.events.emit("transactionsRemoved", [txs]); + // Check that all elements have been removed and were in the mempool prior + // eslint-disable-next-line no-warning-comments,max-len + // TODO Make sure that in case of return value false, it gets rolled back somehow // eslint-disable-next-line unicorn/consistent-destructuring return length === this.queue.length + txs.length; } diff --git a/packages/sequencer/src/protocol/production/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/BlockProducerModule.ts index fbf95469..9e4ae842 100644 --- a/packages/sequencer/src/protocol/production/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/BlockProducerModule.ts @@ -1,36 +1,38 @@ import { inject } from "tsyringe"; import { - AsyncMerkleTreeStore, BlockProverPublicInput, BlockProverPublicOutput, DefaultProvableHashList, NetworkState, ReturnType, } from "@proto-kit/protocol"; -import { Field, Proof, UInt64 } from "o1js"; +import { Field, Proof } from "o1js"; import { log, requireTrue, noop } from "@proto-kit/common"; import { sequencerModule, SequencerModule, } from "../../sequencer/builder/SequencerModule"; -import { Mempool } from "../../mempool/Mempool"; -import { PendingTransaction } from "../../mempool/PendingTransaction"; import { BaseLayer } from "../baselayer/BaseLayer"; import { BlockStorage } from "../../storage/repositories/BlockStorage"; import { ComputedBlock, ComputedBlockTransaction, } from "../../storage/model/Block"; +import { CachedStateService } from "../../state/state/CachedStateService"; +import { CachedMerkleTreeStore } from "../../state/merkle/CachedMerkleTreeStore"; +import { AsyncStateService } from "../../state/async/AsyncStateService"; +import { AsyncMerkleTreeStore } from "../../state/async/AsyncMerkleTreeStore"; -import { AsyncStateService } from "./state/AsyncStateService"; -import { CachedStateService } from "./execution/CachedStateService"; +import { BlockProverParameters } from "./tasks/BlockProvingTask"; import { StateTransitionProofParameters } from "./tasks/StateTransitionTaskParameters"; import { RuntimeProofParameters } from "./tasks/RuntimeTaskParameters"; import { TransactionTraceService } from "./TransactionTraceService"; import { BlockTaskFlowService } from "./BlockTaskFlowService"; -import { BlockProverParameters } from "./tasks/BlockProvingTask"; -import { CachedMerkleTreeStore } from "./execution/CachedMerkleTreeStore"; +import { + TransactionExecutionResult, + UnprovenBlock, +} from "./unproven/TransactionExecutionService"; export interface StateRecord { [key: string]: Field[] | undefined; @@ -49,12 +51,18 @@ interface ComputedBlockMetadata { } const errors = { - txRemovalFailed: () => new Error("Removal of txs from mempool failed"), - blockWithoutTxs: () => new Error("Can't create a block with zero transactions"), }; +export interface BlockProducerModuleConfig { + /** + * Toggles whether on tracing, the block and state transitions provers + * should run a simulation + */ + simulateProvers?: boolean; +} + /** * The BlockProducerModule has the resposiblity to oversee the block production * and combine all necessary parts for that to happen. The flow roughly follows @@ -64,12 +72,10 @@ const errors = { * 2. */ @sequencerModule() -export class BlockProducerModule extends SequencerModule { +export class BlockProducerModule extends SequencerModule { private productionInProgress = false; - // eslint-disable-next-line max-params public constructor( - @inject("Mempool") private readonly mempool: Mempool, @inject("AsyncStateService") private readonly asyncStateService: AsyncStateService, @inject("AsyncMerkleStore") @@ -82,15 +88,10 @@ export class BlockProducerModule extends SequencerModule { super(); } - private createNetworkState(lastHeight: number): NetworkState { - return new NetworkState({ - block: { - height: UInt64.from(lastHeight + 1), - }, - }); - } - - private async applyStateChanges(block: ComputedBlockMetadata) { + private async applyStateChanges( + unprovenBlocks: UnprovenBlock[], + block: ComputedBlockMetadata + ) { await block.stateService.mergeIntoParent(); await block.merkleStore.mergeIntoParent(); } @@ -100,69 +101,94 @@ export class BlockProducerModule extends SequencerModule { * transactions that are present in the mempool. This function should also * be the one called by BlockTriggers */ - public async createBlock(): Promise { + public async createBlock( + unprovenBlocks: UnprovenBlock[] + ): Promise { log.info("Producing batch..."); - const block = await this.tryProduceBlock(); + const blockMetadata = await this.tryProduceBlock(unprovenBlocks); - if (block !== undefined) { - log.debug("Batch produced"); + if (blockMetadata !== undefined) { + log.debug(`Batch produced (${blockMetadata.block.txs.length} txs)`); // Apply state changes to current StateService - await this.applyStateChanges(block); + await this.applyStateChanges(unprovenBlocks, blockMetadata); // Mock for now - await this.blockStorage.pushBlock(block.block); + await this.blockStorage.pushBlock(blockMetadata.block); // Broadcast result on to baselayer - await this.baseLayer.blockProduced(block.block); + await this.baseLayer.blockProduced(blockMetadata.block); log.info("Batch submitted onto baselayer"); } - return block?.block; + return blockMetadata?.block; } public async start(): Promise { noop(); } - private async tryProduceBlock(): Promise { + private async tryProduceBlock( + unprovenBlocks: UnprovenBlock[] + ): Promise { if (!this.productionInProgress) { try { - return await this.produceBlock(); + return await this.produceBlock(unprovenBlocks); } catch (error: unknown) { if (error instanceof Error) { + if ( + !error.message.includes( + "Can't create a block with zero transactions" + ) + ) { + log.error(error); + } + this.productionInProgress = false; throw error; } else { log.error(error); } } + } else { + log.debug( + "Skipping new block production because production is still in progress" + ); } return undefined; } - private async produceBlock(): Promise { + private async produceBlock( + unprovenBlocks: UnprovenBlock[] + ): Promise { this.productionInProgress = true; - // Get next blockheight and therefore taskId - const lastHeight = await this.blockStorage.getCurrentBlockHeight(); - - const { txs } = this.mempool.getTxs(); + // eslint-disable-next-line no-warning-comments + // TODO Workaround for now until transitioning networkstate is implemented + const [{ networkState }] = unprovenBlocks; - const networkState = this.createNetworkState(lastHeight); + const txs = unprovenBlocks.flatMap((block) => block.transactions); - const block = await this.computeBlock(txs, networkState, lastHeight + 1); - - requireTrue(this.mempool.removeTxs(txs), errors.txRemovalFailed); + const block = await this.computeBlock( + txs, + networkState, + Number(networkState.block.height.toBigInt()) + ); this.productionInProgress = false; return { block: { proof: block.proof, - txs: block.computedTransactions, + txs: txs.map((tx) => { + return { + tx: tx.tx, + status: tx.status.toBoolean(), + statusMessage: tx.statusMessage, + }; + }), }, - stateService: block.stateSerivce, + stateService: block.stateService, merkleStore: block.merkleStore, }; } @@ -180,14 +206,13 @@ export class BlockProducerModule extends SequencerModule { * */ private async computeBlock( - txs: PendingTransaction[], + txs: TransactionExecutionResult[], networkState: NetworkState, blockId: number ): Promise<{ proof: Proof; - stateSerivce: CachedStateService; + stateService: CachedStateService; merkleStore: CachedMerkleTreeStore; - computedTransactions: ComputedBlockTransaction[]; }> { if (txs.length === 0) { throw errors.blockWithoutTxs(); @@ -200,9 +225,7 @@ export class BlockProducerModule extends SequencerModule { const bundleTracker = new DefaultProvableHashList(Field); - const traceResults: ReturnType< - typeof TransactionTraceService - >["createTrace"][] = []; + const traces: TransactionTrace[] = []; for (const tx of txs) { // eslint-disable-next-line no-await-in-loop @@ -212,18 +235,15 @@ export class BlockProducerModule extends SequencerModule { networkState, bundleTracker ); - traceResults.push(result); + traces.push(result); } - const traces = traceResults.map((result) => result.trace); - const proof = await this.blockFlowService.executeFlow(traces, blockId); return { proof, - stateSerivce: stateServices.stateService, + stateService: stateServices.stateService, merkleStore: stateServices.merkleStore, - computedTransactions: traceResults.map((result) => result.computedTxs), }; } } diff --git a/packages/sequencer/src/protocol/production/Tracing.md b/packages/sequencer/src/protocol/production/Tracing.md new file mode 100644 index 00000000..238e9472 --- /dev/null +++ b/packages/sequencer/src/protocol/production/Tracing.md @@ -0,0 +1,8 @@ +# Tracing Workflow + +1. Simulate Transaction + +Decode and run the transaction into a result that contains +- ST[] (normal + protocol) +- State diff +- Result status + message \ No newline at end of file diff --git a/packages/sequencer/src/protocol/production/TransactionTraceService.ts b/packages/sequencer/src/protocol/production/TransactionTraceService.ts index 276642dc..f02f8d4b 100644 --- a/packages/sequencer/src/protocol/production/TransactionTraceService.ts +++ b/packages/sequencer/src/protocol/production/TransactionTraceService.ts @@ -1,151 +1,61 @@ /* eslint-disable max-lines,@typescript-eslint/init-declarations */ -import { inject, injectable, Lifecycle, scoped } from "tsyringe"; +import { injectable, Lifecycle, scoped } from "tsyringe"; import { - MethodParameterDecoder, - Runtime, - RuntimeModule, - MethodIdResolver, -} from "@proto-kit/module"; -import { - RuntimeMethodExecutionContext, - RuntimeProvableMethodExecutionResult, - BlockProverExecutionData, DefaultProvableHashList, NetworkState, - Protocol, ProtocolConstants, ProvableHashList, ProvableStateTransition, ProvableStateTransitionType, - ProvableTransactionHook, RollupMerkleTree, - RuntimeTransaction, - StateService, StateTransition, StateTransitionType, - RuntimeMethodExecutionData, } from "@proto-kit/protocol"; import { Bool, Field } from "o1js"; -import { AreProofsEnabled, log } from "@proto-kit/common"; import chunk from "lodash/chunk"; -import { PendingTransaction } from "../../mempool/PendingTransaction"; import { distinctByString } from "../../helpers/utils"; -import { ComputedBlockTransaction } from "../../storage/model/Block"; -import { CachedStateService } from "./execution/CachedStateService"; -import type { StateRecord, TransactionTrace } from "./BlockProducerModule"; -import { DummyStateService } from "./execution/DummyStateService"; +import type { TransactionTrace } from "./BlockProducerModule"; import { StateTransitionProofParameters } from "./tasks/StateTransitionTaskParameters"; -import { AsyncStateService } from "./state/AsyncStateService"; -import { - CachedMerkleTreeStore, - SyncCachedMerkleTreeStore, -} from "./execution/CachedMerkleTreeStore"; - -const errors = { - methodIdNotFound: (methodId: string) => - new Error(`Can't find runtime method with id ${methodId}`), -}; +import { CachedMerkleTreeStore } from "../../state/merkle/CachedMerkleTreeStore"; +import { CachedStateService } from "../../state/state/CachedStateService"; +import { SyncCachedMerkleTreeStore } from "../../state/merkle/SyncCachedMerkleTreeStore"; +import { TransactionExecutionResult } from "./unproven/TransactionExecutionService"; @injectable() @scoped(Lifecycle.ContainerScoped) export class TransactionTraceService { - private readonly dummyStateService = new DummyStateService(); - - private readonly transactionHooks: ProvableTransactionHook[]; - - public constructor( - @inject("Runtime") private readonly runtime: Runtime, - @inject("Protocol") private readonly protocol: Protocol - ) { - this.transactionHooks = protocol.dependencyContainer.resolveAll( - "ProvableTransactionHook" - ); - } - private allKeys(stateTransitions: StateTransition[]): Field[] { // We have to do the distinct with strings because // array.indexOf() doesn't work with fields return stateTransitions.map((st) => st.path).filter(distinctByString); } - private decodeTransaction(tx: PendingTransaction): { - method: (...args: unknown[]) => unknown; - args: unknown[]; - module: RuntimeModule; - } { - const methodDescriptors = this.runtime.dependencyContainer - .resolve("MethodIdResolver") - .getMethodNameFromId(tx.methodId.toBigInt()); - - const method = this.runtime.getMethodById(tx.methodId.toBigInt()); - - if (methodDescriptors === undefined || method === undefined) { - throw errors.methodIdNotFound(tx.methodId.toString()); - } - - const [moduleName, methodName] = methodDescriptors; - const module: RuntimeModule = this.runtime.resolve(moduleName); - - const parameterDecoder = MethodParameterDecoder.fromMethod( - module, - methodName - ); - const args = parameterDecoder.fromFields(tx.args); - - return { - method, - args, - module, - }; - } - - private retrieveStateRecord( - stateService: StateService, - keys: Field[] - ): StateRecord { - // This has to be this detailed bc the CachedStateService collects state - // over the whole block, but we are only interested in the keys touched - // by this tx - return keys - .map<[string, Field[] | undefined]>((key) => [ - key.toString(), - stateService.get(key), - ]) - .reduce((a, b) => { - const [recordKey, value] = b; - a[recordKey] = value; - return a; - }, {}); - } - - private async applyTransitions( + private async collectStartingState( stateService: CachedStateService, stateTransitions: StateTransition[] - ): Promise { - await Promise.all( - // Use updated stateTransitions since only they will have the - // right values - stateTransitions - .filter((st) => st.to.isSome.toBoolean()) - .map(async (st) => { - await stateService.setAsync(st.path, st.to.toFields()); - }) - ); + ): Promise> { + const keys = this.allKeys(stateTransitions); + await stateService.preloadKeys(keys); + + return keys.reduce>((state, key) => { + state[key.toString()] = stateService.get(key); + return state; + }, {}); } - private getAppChainForModule( - module: RuntimeModule - ): AreProofsEnabled { - if (module.runtime === undefined) { - throw new Error("Runtime on RuntimeModule not set"); - } - if (module.runtime.appChain === undefined) { - throw new Error("AppChain on Runtime not set"); - } - const { appChain } = module.runtime; - return appChain; + private applyTransitions( + stateService: CachedStateService, + stateTransitions: StateTransition[] + ): void { + // Use updated stateTransitions since only they will have the + // right values + stateTransitions + .filter((st) => st.to.isSome.toBoolean()) + .forEach((st) => { + stateService.set(st.path, st.to.toFields()); + }); } /** @@ -169,29 +79,31 @@ export class TransactionTraceService { * StateTransitionProveParams */ public async createTrace( - tx: PendingTransaction, + executionResult: TransactionExecutionResult, stateServices: { stateService: CachedStateService; merkleStore: CachedMerkleTreeStore; }, networkState: NetworkState, bundleTracker: ProvableHashList - ): Promise<{ - trace: TransactionTrace; - computedTxs: ComputedBlockTransaction; - }> { - // this.witnessProviderReference.setWitnessProvider( - // new MerkleStoreWitnessProvider(stateServices.merkleStore) - // ); + ): Promise { + const { stateTransitions, protocolTransitions, status, tx } = + executionResult; - // Step 1 & 2 - const { executionResult, startingState } = await this.createExecutionTrace( + // Collect starting state + const protocolStartingState = await this.collectStartingState( stateServices.stateService, - tx, - networkState + protocolTransitions ); - const { stateTransitions, protocolTransitions, status, statusMessage } = - executionResult; + + this.applyTransitions(stateServices.stateService, protocolTransitions); + + const runtimeStartingState = await this.collectStartingState( + stateServices.stateService, + stateTransitions + ); + + this.applyTransitions(stateServices.stateService, stateTransitions); // Step 3 const { stParameters, fromStateRoot } = await this.createMerkleTrace( @@ -204,10 +116,10 @@ export class TransactionTraceService { const transactionsHash = bundleTracker.commitment; bundleTracker.push(tx.hash()); - const trace: TransactionTrace = { + return { runtimeProver: { tx, - state: startingState.runtime, + state: runtimeStartingState, networkState, }, @@ -225,17 +137,7 @@ export class TransactionTraceService { transaction: tx.toProtocolTransaction(), }, - startingState: startingState.protocol, - }, - }; - - return { - trace, - - computedTxs: { - tx, - status: status.toBoolean(), - statusMessage, + startingState: protocolStartingState, }, }; } @@ -366,281 +268,4 @@ export class TransactionTraceService { fromStateRoot: initialRoot, }; } - - private executeRuntimeMethod( - method: (...args: unknown[]) => unknown, - args: unknown[], - contextInputs: RuntimeMethodExecutionData - ): RuntimeProvableMethodExecutionResult { - // Set up context - const executionContext = this.runtime.dependencyContainer.resolve( - RuntimeMethodExecutionContext - ); - executionContext.setup(contextInputs); - - // Execute method - method(...args); - - const runtimeResult = executionContext.current().result; - - // Clear executionContext - executionContext.afterMethod(); - executionContext.clear(); - - return runtimeResult; - } - - /** - * Simulates a certain Context-aware method through multiple rounds. - * - * For a method that emits n Statetransitions, we execute it n times, - * where for every i-th iteration, we collect the i-th ST that has - * been emitted and preload the corresponding key. - */ - private async simulateMultiRound( - method: () => void, - contextInputs: RuntimeMethodExecutionData, - parentStateService: AsyncStateService - ): Promise { - // Set up context - const executionContext = this.runtime.dependencyContainer.resolve( - RuntimeMethodExecutionContext - ); - - let numberMethodSTs: number | undefined; - let collectedSTs = 0; - - const touchedKeys: string[] = []; - - let lastRuntimeResult: RuntimeProvableMethodExecutionResult; - - do { - executionContext.setup(contextInputs); - executionContext.setSimulated(true); - - const stateService = new CachedStateService(parentStateService); - this.runtime.stateServiceProvider.setCurrentStateService(stateService); - this.protocol.stateServiceProvider.setCurrentStateService(stateService); - - // Preload previously determined keys - // eslint-disable-next-line no-await-in-loop - await stateService.preloadKeys( - touchedKeys.map((fieldString) => Field(fieldString)) - ); - - // Execute method - method(); - - lastRuntimeResult = executionContext.current().result; - - // Clear executionContext - executionContext.afterMethod(); - executionContext.clear(); - - const { stateTransitions } = lastRuntimeResult; - - const latestST = stateTransitions.at(collectedSTs); - - if ( - latestST !== undefined && - !touchedKeys.includes(latestST.path.toString()) - ) { - touchedKeys.push(latestST.path.toString()); - } - - if (numberMethodSTs === undefined) { - numberMethodSTs = stateTransitions.length; - } - collectedSTs += 1; - - this.runtime.stateServiceProvider.popCurrentStateService(); - this.protocol.stateServiceProvider.popCurrentStateService(); - } while (collectedSTs < numberMethodSTs); - - return lastRuntimeResult; - } - - private executeProtocolHooks( - runtimeContextInputs: RuntimeMethodExecutionData, - blockContextInputs: BlockProverExecutionData, - runUnchecked = false - ): RuntimeProvableMethodExecutionResult { - // Set up context - const executionContext = this.runtime.dependencyContainer.resolve( - RuntimeMethodExecutionContext - ); - executionContext.setup(runtimeContextInputs); - if (runUnchecked) { - executionContext.setSimulated(true); - } - - this.transactionHooks.forEach((transactionHook) => { - transactionHook.onTransaction(blockContextInputs); - }); - - const protocolResult = executionContext.current().result; - executionContext.afterMethod(); - executionContext.clear(); - - return protocolResult; - } - - private async extractAccessedKeys( - method: (...args: unknown[]) => unknown, - args: unknown[], - runtimeContextInputs: RuntimeMethodExecutionData, - blockContextInputs: BlockProverExecutionData, - parentStateService: AsyncStateService - ): Promise<{ - runtimeKeys: Field[]; - protocolKeys: Field[]; - }> { - // TODO unsafe to re-use params here? - const { stateTransitions } = await this.simulateMultiRound( - () => { - method(...args); - }, - runtimeContextInputs, - parentStateService - ); - - const protocolSimulationResult = await this.simulateMultiRound( - () => { - this.transactionHooks.forEach((transactionHook) => { - transactionHook.onTransaction(blockContextInputs); - }); - }, - runtimeContextInputs, - parentStateService - ); - - const protocolTransitions = protocolSimulationResult.stateTransitions; - - log.debug(`Got ${stateTransitions.length} StateTransitions`); - log.debug(`Got ${protocolTransitions.length} ProtocolStateTransitions`); - - return { - runtimeKeys: this.allKeys(stateTransitions), - protocolKeys: this.allKeys(protocolTransitions), - }; - } - - // eslint-disable-next-line max-statements - private async createExecutionTrace( - stateService: CachedStateService, - tx: PendingTransaction, - networkState: NetworkState - ): Promise<{ - executionResult: { - stateTransitions: StateTransition[]; - protocolTransitions: StateTransition[]; - status: Bool; - statusMessage?: string; - }; - startingState: { - runtime: StateRecord; - protocol: StateRecord; - }; - }> { - const { method, args, module } = this.decodeTransaction(tx); - - // Disable proof generation for tracing - const appChain = this.getAppChainForModule(module); - const previousProofsEnabled = appChain.areProofsEnabled; - appChain.setProofsEnabled(false); - - const blockContextInputs = { - transaction: tx.toProtocolTransaction(), - networkState, - }; - const runtimeContextInputs = { - networkState, - - transaction: RuntimeTransaction.fromProtocolTransaction( - blockContextInputs.transaction - ), - }; - - const { runtimeKeys, protocolKeys } = await this.extractAccessedKeys( - method, - args, - runtimeContextInputs, - blockContextInputs, - stateService - ); - - // Preload keys - await stateService.preloadKeys( - runtimeKeys.concat(protocolKeys).filter(distinctByString) - ); - - // Execute second time with preloaded state. The following steps - // generate and apply the correct STs with the right values - this.runtime.stateServiceProvider.setCurrentStateService(stateService); - this.protocol.stateServiceProvider.setCurrentStateService(stateService); - - // Get starting protocol state - const startingProtocolState = this.retrieveStateRecord( - stateService, - protocolKeys - ); - - const protocolResult = this.executeProtocolHooks( - runtimeContextInputs, - blockContextInputs - ); - - log.debug( - "PSTs:", - protocolResult.stateTransitions.map((x) => x.toJSON()) - ); - - // Apply protocol STs - await this.applyTransitions(stateService, protocolResult.stateTransitions); - - // Now do the same for runtime STs - // Get starting state - const startingRuntimeState = this.retrieveStateRecord( - stateService, - runtimeKeys - ); - - const runtimeResult = this.executeRuntimeMethod( - method, - args, - runtimeContextInputs - ); - - log.debug( - "STs:", - runtimeResult.stateTransitions.map((x) => x.toJSON()) - ); - - // Apply runtime STs (only if the tx succeeded) - if (runtimeResult.status.toBoolean()) { - await this.applyTransitions(stateService, runtimeResult.stateTransitions); - } - - // Reset global stateservice - this.runtime.stateServiceProvider.popCurrentStateService(); - this.protocol.stateServiceProvider.popCurrentStateService(); - // Reset proofs enabled - appChain.setProofsEnabled(previousProofsEnabled); - - return { - executionResult: { - stateTransitions: runtimeResult.stateTransitions, - protocolTransitions: protocolResult.stateTransitions, - status: runtimeResult.status, - statusMessage: runtimeResult.statusMessage, - }, - - startingState: { - // eslint-disable-next-line putout/putout - runtime: startingRuntimeState, - // eslint-disable-next-line putout/putout - protocol: startingProtocolState, - }, - }; - } } diff --git a/packages/sequencer/src/protocol/production/tasks/BlockProvingTask.ts b/packages/sequencer/src/protocol/production/tasks/BlockProvingTask.ts index f32c29a6..439bf142 100644 --- a/packages/sequencer/src/protocol/production/tasks/BlockProvingTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/BlockProvingTask.ts @@ -6,12 +6,11 @@ import { MethodPublicOutput, Protocol, ProtocolModulesRecord, - // eslint-disable-next-line @typescript-eslint/no-shadow ReturnType, StateTransitionProof, StateTransitionProvable, } from "@proto-kit/protocol"; -import { Field, Proof, Provable } from "o1js"; +import { Field, Proof } from "o1js"; import { Runtime } from "@proto-kit/module"; import { inject, injectable, Lifecycle, scoped } from "tsyringe"; import { ProvableMethodExecutionContext } from "@proto-kit/common"; @@ -27,7 +26,7 @@ import { Task } from "../../../worker/flow/Task"; import { CompileRegistry } from "./CompileRegistry"; import { DecodedState, JSONEncodableState } from "./RuntimeTaskParameters"; -import { PreFilledStateService } from "./providers/PreFilledStateService"; +import { PreFilledStateService } from "../../../state/prefilled/PreFilledStateService"; type RuntimeProof = Proof; type BlockProof = Proof; @@ -192,6 +191,22 @@ export class BlockProvingTask ); } + private async executeWithPrefilledStateService( + startingState: DecodedState, + callback: () => Promise + ): Promise { + const prefilledStateService = new PreFilledStateService(startingState); + this.protocol.stateServiceProvider.setCurrentStateService( + prefilledStateService + ); + + const returnValue = await callback(); + + this.protocol.stateServiceProvider.popCurrentStateService(); + + return returnValue; + } + public async compute( input: PairingDerivedInput< StateTransitionProof, @@ -202,23 +217,24 @@ export class BlockProvingTask const stateTransitionProof = input.input1; const runtimeProof = input.input2; - const prefilledStateService = new PreFilledStateService( - input.params.startingState - ); - this.protocol.stateServiceProvider.setCurrentStateService( - prefilledStateService + await this.executeWithPrefilledStateService( + input.params.startingState, + // eslint-disable-next-line putout/putout + async () => { + this.blockProver.proveTransaction( + input.params.publicInput, + stateTransitionProof, + runtimeProof, + input.params.executionData + ); + } ); - this.blockProver.proveTransaction( - input.params.publicInput, - stateTransitionProof, - runtimeProof, - input.params.executionData + return await this.executeWithPrefilledStateService( + input.params.startingState, + async () => + await this.executionContext.current().result.prove() ); - - this.protocol.stateServiceProvider.popCurrentStateService(); - - return await this.executionContext.current().result.prove(); } // eslint-disable-next-line sonarjs/no-identical-functions diff --git a/packages/sequencer/src/protocol/production/tasks/RuntimeProvingTask.ts b/packages/sequencer/src/protocol/production/tasks/RuntimeProvingTask.ts index 6ae989a0..1b14ff38 100644 --- a/packages/sequencer/src/protocol/production/tasks/RuntimeProvingTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/RuntimeProvingTask.ts @@ -19,7 +19,7 @@ import { RuntimeProofParameters, RuntimeProofParametersSerializer, } from "./RuntimeTaskParameters"; -import { PreFilledStateService } from "./providers/PreFilledStateService"; +import { PreFilledStateService } from "../../../state/prefilled/PreFilledStateService"; type RuntimeProof = Proof; diff --git a/packages/sequencer/src/protocol/production/tasks/RuntimeTaskParameters.ts b/packages/sequencer/src/protocol/production/tasks/RuntimeTaskParameters.ts index 1ac020f8..c6f1830d 100644 --- a/packages/sequencer/src/protocol/production/tasks/RuntimeTaskParameters.ts +++ b/packages/sequencer/src/protocol/production/tasks/RuntimeTaskParameters.ts @@ -4,9 +4,7 @@ import { NetworkState, ReturnType } from "@proto-kit/protocol"; import { PendingTransaction } from "../../../mempool/PendingTransaction"; import { TaskSerializer } from "../../../worker/manager/ReducableTask"; -export interface DecodedState { - [key: string]: Field[] | undefined; -} +export type DecodedState = Record; export interface RuntimeProofParameters { // publicInput: MethodPublicInput; diff --git a/packages/sequencer/src/protocol/production/tasks/StateTransitionTask.ts b/packages/sequencer/src/protocol/production/tasks/StateTransitionTask.ts index d74801c2..42ee610a 100644 --- a/packages/sequencer/src/protocol/production/tasks/StateTransitionTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/StateTransitionTask.ts @@ -25,7 +25,7 @@ import { StateTransitionParametersSerializer, StateTransitionProofParameters, } from "./StateTransitionTaskParameters"; -import { PreFilledWitnessProvider } from "./providers/PreFilledWitnessProvider"; +import { PreFilledWitnessProvider } from "../../../state/prefilled/PreFilledWitnessProvider"; import { CompileRegistry } from "./CompileRegistry"; @injectable() diff --git a/packages/sequencer/src/protocol/production/trigger/AutomaticBlockTrigger.ts b/packages/sequencer/src/protocol/production/trigger/AutomaticBlockTrigger.ts new file mode 100644 index 00000000..683af37c --- /dev/null +++ b/packages/sequencer/src/protocol/production/trigger/AutomaticBlockTrigger.ts @@ -0,0 +1,48 @@ +import { inject } from "tsyringe"; +import { log } from "@proto-kit/common"; + +import { SequencerModule } from "../../../sequencer/builder/SequencerModule"; +import { UnprovenProducerModule } from "../unproven/UnprovenProducerModule"; +import { Mempool } from "../../../mempool/Mempool"; + +import { BlockTrigger } from "./BlockTrigger"; + +/** + * Only unproven invocation at the moment, because + * this is primarily for development and testing purposes + */ +export class AutomaticBlockTrigger + extends SequencerModule> + implements BlockTrigger +{ + public constructor( + @inject("UnprovenProducerModule") + private readonly unprovenProducerModule: UnprovenProducerModule, + @inject("Mempool") + private readonly mempool: Mempool + ) { + super(); + } + + public async start(): Promise { + // eslint-disable-next-line @typescript-eslint/no-misused-promises + this.mempool.events.on("transactionAdded", async () => { + log.info("Transaction received, creating block..."); + const block = await this.unprovenProducerModule.tryProduceUnprovenBlock(); + + // In case the block producer was busy, we need to re-trigger production + // as soon as the previous production was finished + if (block === undefined) { + this.unprovenProducerModule.events.on( + "unprovenBlockProduced", + async () => { + // eslint-disable-next-line max-len + // Make sure this comes before await, because otherwise we have a race condition + this.unprovenProducerModule.events.offSelf(); + await this.unprovenProducerModule.tryProduceUnprovenBlock(); + } + ); + } + }); + } +} diff --git a/packages/sequencer/src/protocol/production/trigger/ManualBlockTrigger.ts b/packages/sequencer/src/protocol/production/trigger/ManualBlockTrigger.ts index 2c2a1317..9f24bc95 100644 --- a/packages/sequencer/src/protocol/production/trigger/ManualBlockTrigger.ts +++ b/packages/sequencer/src/protocol/production/trigger/ManualBlockTrigger.ts @@ -4,6 +4,9 @@ import { noop } from "@proto-kit/common"; import { SequencerModule } from "../../../sequencer/builder/SequencerModule"; import { ComputedBlock } from "../../../storage/model/Block"; import { BlockProducerModule } from "../BlockProducerModule"; +import { UnprovenProducerModule } from "../unproven/UnprovenProducerModule"; +import { UnprovenBlockQueue } from "../../../storage/repositories/UnprovenBlockStorage"; +import { UnprovenBlock } from "../unproven/TransactionExecutionService"; import { BlockTrigger } from "./BlockTrigger"; @@ -14,13 +17,43 @@ export class ManualBlockTrigger { public constructor( @inject("BlockProducerModule") - private readonly blockProducerModule: BlockProducerModule + private readonly blockProducerModule: BlockProducerModule, + @inject("UnprovenProducerModule") + private readonly unprovenProducerModule: UnprovenProducerModule, + @inject("UnprovenBlockQueue") + private readonly unprovenBlockQueue: UnprovenBlockQueue ) { super(); } + /** + * Produces both an unproven block and immediately produce a + * settlement block proof + */ public async produceBlock(): Promise { - return await this.blockProducerModule.createBlock(); + await this.produceUnproven(); + return await this.produceProven(); + } + + public async produceProven(): Promise { + const blocks = await this.unprovenBlockQueue.popNewBlocks(true); + if (blocks.length > 0) { + return await this.blockProducerModule.createBlock(blocks); + } + return undefined; + } + + public async produceUnproven( + enqueueInSettlementQueue = true + ): Promise { + const unprovenBlock = + await this.unprovenProducerModule.tryProduceUnprovenBlock(); + + if (unprovenBlock && enqueueInSettlementQueue) { + await this.unprovenBlockQueue.pushBlock(unprovenBlock); + } + + return unprovenBlock; } public async start(): Promise { diff --git a/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts b/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts index 89f1730b..79688aaa 100644 --- a/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts +++ b/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts @@ -6,14 +6,19 @@ import { BlockProducerModule } from "../BlockProducerModule"; import { Mempool } from "../../../mempool/Mempool"; import { BlockTrigger } from "./BlockTrigger"; +import { UnprovenProducerModule } from "../unproven/UnprovenProducerModule"; +import { gcd, log } from "@proto-kit/common"; +import { UnprovenBlockQueue } from "../../../storage/repositories/UnprovenBlockStorage"; -export interface BlockTimeConfig { - blocktime: number; +export interface TimedBlockTriggerConfig { + settlementInterval?: number; + blockInterval: number; + produceEmptyBlocks?: boolean; } @injectable() export class TimedBlockTrigger - extends SequencerModule + extends SequencerModule implements BlockTrigger, Closeable { // There is no real type for interval ids somehow, so any it is @@ -23,6 +28,10 @@ export class TimedBlockTrigger public constructor( @inject("BlockProducerModule") private readonly blockProducerModule: BlockProducerModule, + @inject("UnprovenProducerModule") + private readonly unprovenProducerModule: UnprovenProducerModule, + @inject("UnprovenBlockQueue") + private readonly unprovenBlockQueue: UnprovenBlockQueue, @inject("Mempool") private readonly mempool: Mempool ) { @@ -30,11 +39,57 @@ export class TimedBlockTrigger } public async start(): Promise { - this.interval = setInterval(() => { - if (this.mempool.getTxs().txs.length > 0) { - void this.blockProducerModule.createBlock(); + const { settlementInterval, blockInterval } = this.config; + + const timerInterval = + settlementInterval !== undefined + ? gcd(settlementInterval, blockInterval) + : blockInterval; + + let totalTime = 0; + this.interval = setInterval(async () => { + totalTime += timerInterval; + + try { + // Trigger unproven blocks + if (totalTime % blockInterval === 0) { + await this.produceUnprovenBlock(); + } + + // Trigger proven (settlement) blocks + // Only produce settlements if a time has been set + // otherwise treat as unproven-only + if ( + settlementInterval !== undefined && + totalTime % settlementInterval === 0 + ) { + await this.produceBlock(); + } + } catch (error) { + log.error(error); } - }, this.config.blocktime); + }, timerInterval); + } + + private async produceUnprovenBlock() { + // Produce a block if either produceEmptyBlocks is true or we have more + // than 1 tx in mempool + if ( + this.mempool.getTxs().txs.length > 0 || + (this.config.produceEmptyBlocks ?? true) + ) { + const block = await this.unprovenProducerModule.tryProduceUnprovenBlock(); + if (block !== undefined) { + await this.unprovenBlockQueue.pushBlock(block); + } + } + } + + private async produceBlock() { + const unprovenBlocks = await this.unprovenBlockQueue.popNewBlocks(true); + if (unprovenBlocks.length > 0) { + void this.blockProducerModule.createBlock(unprovenBlocks); + } } public async close(): Promise { diff --git a/packages/sequencer/src/protocol/production/unproven/RuntimeMethodExecution.ts b/packages/sequencer/src/protocol/production/unproven/RuntimeMethodExecution.ts new file mode 100644 index 00000000..68099e7a --- /dev/null +++ b/packages/sequencer/src/protocol/production/unproven/RuntimeMethodExecution.ts @@ -0,0 +1,147 @@ +import { + RuntimeMethodExecutionContext, + RuntimeMethodExecutionData, + StateTransition, +} from "@proto-kit/protocol"; +import { AsyncStateService } from "../../../state/async/AsyncStateService"; +import { CachedStateService } from "../../../state/state/CachedStateService"; +import { Field } from "o1js"; +import { RuntimeEnvironment } from "@proto-kit/module/dist/runtime/RuntimeEnvironment"; +import { ProtocolEnvironment } from "@proto-kit/protocol/dist/protocol/ProtocolEnvironment"; +import { distinctByString } from "../../../helpers/utils"; +import _ from "lodash"; + +export class RuntimeMethodExecution { + public constructor( + private readonly runtime: RuntimeEnvironment, + private readonly protocol: ProtocolEnvironment, + private readonly executionContext: RuntimeMethodExecutionContext + ) {} + + private async executeMethodWithKeys( + touchedKeys: string[], + method: () => void, + contextInputs: RuntimeMethodExecutionData, + parentStateService: AsyncStateService + ): Promise[]> { + const { executionContext } = this; + + executionContext.setup(contextInputs); + executionContext.setSimulated(true); + + const stateService = new CachedStateService(parentStateService); + this.runtime.stateServiceProvider.setCurrentStateService(stateService); + this.protocol.stateServiceProvider.setCurrentStateService(stateService); + + // Preload previously determined keys + // eslint-disable-next-line no-await-in-loop + await stateService.preloadKeys( + touchedKeys.map((fieldString) => Field(fieldString)) + ); + + // Execute method + method(); + + const lastRuntimeResult = executionContext.current().result; + + // Clear executionContext + executionContext.afterMethod(); + executionContext.clear(); + + this.runtime.stateServiceProvider.popCurrentStateService(); + this.protocol.stateServiceProvider.popCurrentStateService(); + + const { stateTransitions } = lastRuntimeResult; + + return stateTransitions; + } + + /** + * Simulates a certain Context-aware method through multiple rounds. + * + * For a method that emits n Statetransitions, we execute it n times, + * where for every i-th iteration, we collect the i-th ST that has + * been emitted and preload the corresponding key. + */ + public async simulateMultiRound( + method: () => void, + contextInputs: RuntimeMethodExecutionData, + parentStateService: AsyncStateService + ): Promise[]> { + // Set up context + const executionContext = this.executionContext; + + let numberMethodSTs: number | undefined; + let collectedSTs = 0; + + const touchedKeys: string[] = []; + + let lastRuntimeResult: StateTransition[]; + + do { + const stateTransitions = await this.executeMethodWithKeys( + touchedKeys, + method, + contextInputs, + parentStateService + ); + + if (numberMethodSTs === undefined) { + numberMethodSTs = stateTransitions.length; + } + + if (collectedSTs === 0) { + // Do a full run with all keys and see if keys have changed + // (i.e. if no dynamic keys are used, just take that result) + // If that is the case, fast-forward to the first dynamic key + const keys = stateTransitions + .map((st) => st.path.toString()) + .filter(distinctByString); + const stateTransitionsFullRun = await this.executeMethodWithKeys( + keys, + method, + contextInputs, + parentStateService + ); + + const firstDiffIndex = _.zip( + stateTransitions, + stateTransitionsFullRun + ).findIndex( + ([st1, st2]) => st1?.path.toString() !== st2?.path.toString() + ); + + if (firstDiffIndex === -1) { + // Abort bcs no dynamic keys are used => use then 1:1 + return stateTransitionsFullRun; + } else { + // here push all known keys up to the first dynamic key + // touchedkeys is empty, so we don't care about that + const additionalKeys = stateTransitionsFullRun + .slice(0, firstDiffIndex) + .map((st) => st.path.toString()) + .filter(distinctByString); + touchedKeys.push(...additionalKeys); + collectedSTs = firstDiffIndex - 1; + lastRuntimeResult = stateTransitions; + continue; + } + } + + const latestST = stateTransitions.at(collectedSTs); + + if ( + latestST !== undefined && + !touchedKeys.includes(latestST.path.toString()) + ) { + touchedKeys.push(latestST.path.toString()); + } + + collectedSTs += 1; + + lastRuntimeResult = stateTransitions; + } while (collectedSTs < numberMethodSTs); + + return lastRuntimeResult; + } +} diff --git a/packages/sequencer/src/protocol/production/unproven/TransactionExecutionService.ts b/packages/sequencer/src/protocol/production/unproven/TransactionExecutionService.ts new file mode 100644 index 00000000..a3315cb5 --- /dev/null +++ b/packages/sequencer/src/protocol/production/unproven/TransactionExecutionService.ts @@ -0,0 +1,382 @@ +import { inject, injectable, Lifecycle, scoped } from "tsyringe"; +import { + BlockProverExecutionData, + NetworkState, + Protocol, + ProtocolModulesRecord, + ProvableTransactionHook, + RuntimeMethodExecutionContext, + RuntimeMethodExecutionData, + RuntimeProvableMethodExecutionResult, + RuntimeTransaction, + StateTransition, +} from "@proto-kit/protocol"; +import { Bool, Field } from "o1js"; +import { AreProofsEnabled, log } from "@proto-kit/common"; +import { + MethodParameterDecoder, + Runtime, + RuntimeModule, + RuntimeModulesRecord, +} from "@proto-kit/module"; + +import { PendingTransaction } from "../../../mempool/PendingTransaction"; +import { CachedStateService } from "../../../state/state/CachedStateService"; +import { StateRecord } from "../BlockProducerModule"; +import { distinctByString } from "../../../helpers/utils"; +import { AsyncStateService } from "../../../state/async/AsyncStateService"; + +import { RuntimeMethodExecution } from "./RuntimeMethodExecution"; + +const errors = { + methodIdNotFound: (methodId: string) => + new Error(`Can't find runtime method with id ${methodId}`), +}; + +export interface TransactionExecutionResult { + tx: PendingTransaction; + stateTransitions: StateTransition[]; + protocolTransitions: StateTransition[]; + status: Bool; + statusMessage?: string; + stateDiff: StateRecord; +} + +export interface UnprovenBlock { + networkState: NetworkState; + transactions: TransactionExecutionResult[]; +} + +@injectable() +@scoped(Lifecycle.ContainerScoped) +export class TransactionExecutionService { + private readonly transactionHooks: ProvableTransactionHook[]; + + private readonly runtimeMethodExecution: RuntimeMethodExecution; + + public constructor( + @inject("Runtime") private readonly runtime: Runtime, + @inject("Protocol") + private readonly protocol: Protocol + ) { + this.transactionHooks = protocol.dependencyContainer.resolveAll( + "ProvableTransactionHook" + ); + + this.runtimeMethodExecution = new RuntimeMethodExecution( + this.runtime, + this.protocol, + this.runtime.dependencyContainer.resolve(RuntimeMethodExecutionContext) + ); + } + + private allKeys(stateTransitions: StateTransition[]): Field[] { + // We have to do the distinct with strings because + // array.indexOf() doesn't work with fields + return stateTransitions.map((st) => st.path).filter(distinctByString); + } + + private decodeTransaction(tx: PendingTransaction): { + method: (...args: unknown[]) => unknown; + args: unknown[]; + module: RuntimeModule; + } { + const methodDescriptors = this.runtime.methodIdResolver.getMethodNameFromId( + tx.methodId.toBigInt() + ); + + const method = this.runtime.getMethodById(tx.methodId.toBigInt()); + + if (methodDescriptors === undefined || method === undefined) { + throw errors.methodIdNotFound(tx.methodId.toString()); + } + + const [moduleName, methodName] = methodDescriptors; + const module: RuntimeModule = this.runtime.resolve(moduleName); + + const parameterDecoder = MethodParameterDecoder.fromMethod( + module, + methodName + ); + const args = parameterDecoder.fromFields(tx.args); + + return { + method, + args, + module, + }; + } + + private getAppChainForModule( + module: RuntimeModule + ): AreProofsEnabled { + if (module.runtime === undefined) { + throw new Error("Runtime on RuntimeModule not set"); + } + if (module.runtime.appChain === undefined) { + throw new Error("AppChain on Runtime not set"); + } + const { appChain } = module.runtime; + return appChain; + } + + private executeRuntimeMethod( + method: (...args: unknown[]) => unknown, + args: unknown[], + contextInputs: RuntimeMethodExecutionData + ): RuntimeProvableMethodExecutionResult { + // Set up context + const executionContext = this.runtime.dependencyContainer.resolve( + RuntimeMethodExecutionContext + ); + executionContext.setup(contextInputs); + + // Execute method + method(...args); + + const runtimeResult = executionContext.current().result; + + // Clear executionContext + executionContext.afterMethod(); + executionContext.clear(); + + return runtimeResult; + } + + private executeProtocolHooks( + runtimeContextInputs: RuntimeMethodExecutionData, + blockContextInputs: BlockProverExecutionData, + runUnchecked = false + ): RuntimeProvableMethodExecutionResult { + // Set up context + const executionContext = this.runtime.dependencyContainer.resolve( + RuntimeMethodExecutionContext + ); + executionContext.setup(runtimeContextInputs); + if (runUnchecked) { + executionContext.setSimulated(true); + } + + this.transactionHooks.forEach((transactionHook) => { + transactionHook.onTransaction(blockContextInputs); + }); + + const protocolResult = executionContext.current().result; + executionContext.afterMethod(); + executionContext.clear(); + + return protocolResult; + } + + /** + * Main entry point for creating a unproven block with everything + * attached that is needed for tracing + */ + public async createUnprovenBlock( + stateService: CachedStateService, + transactions: PendingTransaction[], + networkState: NetworkState + ): Promise { + const executionResults: TransactionExecutionResult[] = []; + + for (const tx of transactions) { + try { + // eslint-disable-next-line no-await-in-loop + const executionTrace = await this.createExecutionTrace( + stateService, + tx, + networkState + ); + executionResults.push(executionTrace); + } catch (error) { + if (error instanceof Error) { + log.error("Error in inclusion of tx, skipping", error); + } + } + } + + return { + transactions: executionResults, + networkState, + }; + } + + private collectStateDiff( + stateService: CachedStateService, + stateTransitions: StateTransition[] + ): StateRecord { + const keys = this.allKeys(stateTransitions); + + return keys.reduce>((state, key) => { + state[key.toString()] = stateService.get(key); + return state; + }, {}); + } + + private async applyTransitions( + stateService: CachedStateService, + stateTransitions: StateTransition[] + ): Promise { + await Promise.all( + // Use updated stateTransitions since only they will have the + // right values + stateTransitions + .filter((st) => st.to.isSome.toBoolean()) + .map(async (st) => { + await stateService.setAsync(st.path, st.to.toFields()); + }) + ); + } + + // eslint-disable-next-line no-warning-comments + // TODO Here exists a edge-case, where the protocol hooks set + // some state that is then consumed by the runtime and used as a key. + // In this case, runtime would generate a wrong key here. + private async extractAccessedKeys( + method: (...args: unknown[]) => unknown, + args: unknown[], + runtimeContextInputs: RuntimeMethodExecutionData, + blockContextInputs: BlockProverExecutionData, + parentStateService: AsyncStateService + ): Promise<{ + runtimeKeys: Field[]; + protocolKeys: Field[]; + }> { + // eslint-disable-next-line no-warning-comments + // TODO unsafe to re-use params here? + const stateTransitions = + await this.runtimeMethodExecution.simulateMultiRound( + () => { + method(...args); + }, + runtimeContextInputs, + parentStateService + ); + + const protocolTransitions = + await this.runtimeMethodExecution.simulateMultiRound( + () => { + this.transactionHooks.forEach((transactionHook) => { + transactionHook.onTransaction(blockContextInputs); + }); + }, + runtimeContextInputs, + parentStateService + ); + + log.debug(`Got ${stateTransitions.length} StateTransitions`); + log.debug(`Got ${protocolTransitions.length} ProtocolStateTransitions`); + + return { + runtimeKeys: this.allKeys(stateTransitions), + protocolKeys: this.allKeys(protocolTransitions), + }; + } + + // eslint-disable-next-line max-statements + private async createExecutionTrace( + stateService: CachedStateService, + tx: PendingTransaction, + networkState: NetworkState + ): Promise { + const { method, args, module } = this.decodeTransaction(tx); + + // Disable proof generation for tracing + const appChain = this.getAppChainForModule(module); + const previousProofsEnabled = appChain.areProofsEnabled; + appChain.setProofsEnabled(false); + + const blockContextInputs = { + transaction: tx.toProtocolTransaction(), + networkState, + }; + const runtimeContextInputs = { + networkState, + + transaction: RuntimeTransaction.fromProtocolTransaction( + blockContextInputs.transaction + ), + }; + + const { runtimeKeys, protocolKeys } = await this.extractAccessedKeys( + method, + args, + runtimeContextInputs, + blockContextInputs, + stateService + ); + + // Preload keys + await stateService.preloadKeys( + runtimeKeys.concat(protocolKeys).filter(distinctByString) + ); + + // Execute second time with preloaded state. The following steps + // generate and apply the correct STs with the right values + this.runtime.stateServiceProvider.setCurrentStateService(stateService); + this.protocol.stateServiceProvider.setCurrentStateService(stateService); + + const protocolResult = this.executeProtocolHooks( + runtimeContextInputs, + blockContextInputs + ); + + if (!protocolResult.status.toBoolean()) { + throw new Error( + `Protocol hooks not executable: ${ + protocolResult.statusMessage ?? "unknown" + }` + ); + } + + log.debug( + "PSTs:", + protocolResult.stateTransitions.map((x) => x.toJSON()) + ); + + // Apply protocol STs + await this.applyTransitions(stateService, protocolResult.stateTransitions); + + let stateDiff = this.collectStateDiff( + stateService, + protocolResult.stateTransitions + ); + + const runtimeResult = this.executeRuntimeMethod( + method, + args, + runtimeContextInputs + ); + + log.debug( + "STs:", + runtimeResult.stateTransitions.map((x) => x.toJSON()) + ); + + // Apply runtime STs (only if the tx succeeded) + if (runtimeResult.status.toBoolean()) { + await this.applyTransitions(stateService, runtimeResult.stateTransitions); + + stateDiff = this.collectStateDiff( + stateService, + protocolResult.stateTransitions.concat(runtimeResult.stateTransitions) + ); + } + + // Reset global stateservice + this.runtime.stateServiceProvider.popCurrentStateService(); + this.protocol.stateServiceProvider.popCurrentStateService(); + // Reset proofs enabled + appChain.setProofsEnabled(previousProofsEnabled); + + return { + tx, + stateTransitions: runtimeResult.stateTransitions, + protocolTransitions: protocolResult.stateTransitions, + status: runtimeResult.status, + statusMessage: runtimeResult.statusMessage, + + stateDiff, + }; + } +} diff --git a/packages/sequencer/src/protocol/production/unproven/UnprovenProducerModule.ts b/packages/sequencer/src/protocol/production/unproven/UnprovenProducerModule.ts new file mode 100644 index 00000000..086eb01f --- /dev/null +++ b/packages/sequencer/src/protocol/production/unproven/UnprovenProducerModule.ts @@ -0,0 +1,110 @@ +import { inject } from "tsyringe"; +import { NetworkState } from "@proto-kit/protocol"; +import { UInt64 } from "o1js"; +import { + EventEmitter, + EventEmittingComponent, + EventsRecord, + log, + noop, + requireTrue, +} from "@proto-kit/common"; + +import { Mempool } from "../../../mempool/Mempool"; +import { BlockStorage } from "../../../storage/repositories/BlockStorage"; +import { CachedStateService } from "../../../state/state/CachedStateService"; +import { + sequencerModule, + SequencerModule, +} from "../../../sequencer/builder/SequencerModule"; + +import { + TransactionExecutionService, + UnprovenBlock, +} from "./TransactionExecutionService"; + +const errors = { + txRemovalFailed: () => new Error("Removal of txs from mempool failed"), +}; + +interface UnprovenProducerEvents extends EventsRecord { + unprovenBlockProduced: [UnprovenBlock]; +} + +@sequencerModule() +export class UnprovenProducerModule + extends SequencerModule + implements EventEmittingComponent +{ + private productionInProgress = false; + + public events = new EventEmitter(); + + public constructor( + @inject("Mempool") private readonly mempool: Mempool, + @inject("BlockStorage") private readonly blockStorage: BlockStorage, + @inject("UnprovenStateService") + private readonly unprovenStateService: CachedStateService, + private readonly executionService: TransactionExecutionService + ) { + super(); + } + + private createNetworkState(lastHeight: number): NetworkState { + return new NetworkState({ + block: { + height: UInt64.from(lastHeight + 1), + }, + }); + } + + public async tryProduceUnprovenBlock(): Promise { + if (!this.productionInProgress) { + try { + const block = await this.produceUnprovenBlock(); + + log.info(`Produced unproven block (${block.transactions.length} txs)`); + this.events.emit("unprovenBlockProduced", [block]); + + return block; + } catch (error: unknown) { + if (error instanceof Error) { + this.productionInProgress = false; + throw error; + } else { + log.error(error); + } + } + } + return undefined; + } + + private async produceUnprovenBlock(): Promise { + this.productionInProgress = true; + + // Get next blockheight and therefore taskId + const lastHeight = await this.blockStorage.getCurrentBlockHeight(); + + const { txs } = this.mempool.getTxs(); + + const networkState = this.createNetworkState(lastHeight); + + const stateService = new CachedStateService(this.unprovenStateService); + + const block = await this.executionService.createUnprovenBlock( + stateService, + txs, + networkState + ); + + this.productionInProgress = false; + + requireTrue(this.mempool.removeTxs(txs), errors.txRemovalFailed); + + return block; + } + + public async start() { + noop(); + } +} diff --git a/packages/sequencer/src/sequencer/executor/Sequencer.ts b/packages/sequencer/src/sequencer/executor/Sequencer.ts index 2b64cc7d..8c91e77c 100644 --- a/packages/sequencer/src/sequencer/executor/Sequencer.ts +++ b/packages/sequencer/src/sequencer/executor/Sequencer.ts @@ -13,6 +13,7 @@ import { DependencyContainer, injectable } from "tsyringe"; import { SequencerModule } from "../builder/SequencerModule"; import { Sequenceable } from "./Sequenceable"; +import { MethodIdFactory } from "@proto-kit/module/dist/factories/MethodIdFactory"; export type SequencerModulesRecord = ModulesRecord< TypedClass> @@ -68,6 +69,8 @@ export class Sequencer // ); // witnessProviderReference.setWitnessProvider(witnessProvider); + this.registerDependencyFactories([MethodIdFactory]); + // Log startup info const moduleClassNames = Object.values(this.definition.modules).map( (clazz) => clazz.name diff --git a/packages/sequencer/src/protocol/production/execution/MerkleStoreWitnessProvider.ts b/packages/sequencer/src/state/MerkleStoreWitnessProvider.ts similarity index 100% rename from packages/sequencer/src/protocol/production/execution/MerkleStoreWitnessProvider.ts rename to packages/sequencer/src/state/MerkleStoreWitnessProvider.ts diff --git a/packages/sequencer/src/state/StateServices.md b/packages/sequencer/src/state/StateServices.md new file mode 100644 index 00000000..9802dd64 --- /dev/null +++ b/packages/sequencer/src/state/StateServices.md @@ -0,0 +1,32 @@ +## State Service Architecture + +The state services are built using a abstraction stack, where every service that is below the root is based on the parent. +All of this services have the ability to base off one another to allow for diff-based changes. +You can think of it a bit like git, where every commit is only the diff between the parents state and the next state. + +*But*, in this architecture there is no such thing as branching, it is strictly linear and behaves almost like a traditional stack. +That means, for example, that you shouldn't change state in a service that has one or more children, as these changes might not get reflected instantly in the children's perception of state. + +Now I will go over all the different kinds of stateservices + +Btw all of this also applies to to merkle tree stores + +### AsyncStateService + +This is always the base for the whole stack and is meant to be implemented by the actual persistence layer. +That means primarily the DB (but is also implemented by the in-memory state service). +It's functions are async-based in order to enable external DB APIs + +### CachedStateService + +It receives a AsyncStateService as a parent and can build on top of it. +It "caches" in the sense that it can preload state entries from the parent (asynchronously) and then lets circuits among others perform synchronous operations on them. +It additionally caches write operations that can then later be merged back into the parent (or thrown away). + +### SyncCachedStateService + +These are the same as CachedStateService, with the difference that it accepts a CachedStateService and requires no preloading. + +### PreFilledStateService + +Pre-filled with only a part of the data needed. This is mostly used in Task implementation where all state needed is passed as args. \ No newline at end of file diff --git a/packages/sequencer/src/state/async/AsyncMerkleTreeStore.ts b/packages/sequencer/src/state/async/AsyncMerkleTreeStore.ts new file mode 100644 index 00000000..30e0b8cd --- /dev/null +++ b/packages/sequencer/src/state/async/AsyncMerkleTreeStore.ts @@ -0,0 +1,9 @@ +export interface AsyncMerkleTreeStore { + openTransaction: () => void; + + commit: () => void; + + setNodeAsync: (key: bigint, level: number, value: bigint) => Promise; + + getNodeAsync: (key: bigint, level: number) => Promise; +} \ No newline at end of file diff --git a/packages/sequencer/src/protocol/production/state/AsyncStateService.ts b/packages/sequencer/src/state/async/AsyncStateService.ts similarity index 88% rename from packages/sequencer/src/protocol/production/state/AsyncStateService.ts rename to packages/sequencer/src/state/async/AsyncStateService.ts index 9b00eefe..fe12fe84 100644 --- a/packages/sequencer/src/protocol/production/state/AsyncStateService.ts +++ b/packages/sequencer/src/state/async/AsyncStateService.ts @@ -6,6 +6,11 @@ import { Field } from "o1js"; * CachedStateService to preload keys for In-Circuit usage. */ export interface AsyncStateService { + openTransaction: () => void; + + commit: () => void; + setAsync: (key: Field, value: Field[] | undefined) => Promise; + getAsync: (key: Field) => Promise; } diff --git a/packages/sequencer/src/protocol/production/execution/CachedMerkleTreeStore.ts b/packages/sequencer/src/state/merkle/CachedMerkleTreeStore.ts similarity index 76% rename from packages/sequencer/src/protocol/production/execution/CachedMerkleTreeStore.ts rename to packages/sequencer/src/state/merkle/CachedMerkleTreeStore.ts index 72102b62..dcdabcbe 100644 --- a/packages/sequencer/src/protocol/production/execution/CachedMerkleTreeStore.ts +++ b/packages/sequencer/src/state/merkle/CachedMerkleTreeStore.ts @@ -1,9 +1,9 @@ import { - AsyncMerkleTreeStore, - InMemoryMerkleTreeStorage, MerkleTreeStore, + InMemoryMerkleTreeStorage, RollupMerkleTree } from "@proto-kit/protocol"; import { log, noop } from "@proto-kit/common"; +import { AsyncMerkleTreeStore } from "../async/AsyncMerkleTreeStore"; export class CachedMerkleTreeStore extends InMemoryMerkleTreeStorage @@ -123,42 +123,4 @@ export class CachedMerkleTreeStore this.getNode(key, level) ?? (await this.parent.getNodeAsync(key, level)) ); } -} - -export class SyncCachedMerkleTreeStore extends InMemoryMerkleTreeStorage { - private writeCache: { - [key: number]: { - [key: string]: bigint; - }; - } = {}; - - public constructor(private readonly parent: MerkleTreeStore) { - super(); - } - - public getNode(key: bigint, level: number): bigint | undefined { - return super.getNode(key, level) ?? this.parent.getNode(key, level); - } - - public setNode(key: bigint, level: number, value: bigint) { - super.setNode(key, level, value); - (this.writeCache[level] ??= {})[key.toString()] = value; - } - - public mergeIntoParent() { - if (Object.keys(this.writeCache).length === 0) { - return; - } - - const { height } = RollupMerkleTree; - const nodes = this.writeCache - - Array.from({ length: height }).forEach((ignored, level) => - Object.entries(nodes[level]).forEach((entry) => { - this.parent.setNode(BigInt(entry[0]), level, entry[1]); - }) - ); - - this.writeCache = {} - } } \ No newline at end of file diff --git a/packages/sequencer/src/state/merkle/SyncCachedMerkleTreeStore.ts b/packages/sequencer/src/state/merkle/SyncCachedMerkleTreeStore.ts new file mode 100644 index 00000000..db17bd08 --- /dev/null +++ b/packages/sequencer/src/state/merkle/SyncCachedMerkleTreeStore.ts @@ -0,0 +1,35 @@ +import { + InMemoryMerkleTreeStorage, MerkleTreeStore, + RollupMerkleTree +} from "@proto-kit/protocol"; + +export class SyncCachedMerkleTreeStore extends InMemoryMerkleTreeStorage { + public constructor(private readonly parent: MerkleTreeStore) { + super(); + } + + public getNode(key: bigint, level: number): bigint | undefined { + return super.getNode(key, level) ?? this.parent.getNode(key, level); + } + + public setNode(key: bigint, level: number, value: bigint) { + super.setNode(key, level, value); + } + + public mergeIntoParent() { + if (Object.keys(this.nodes).length === 0) { + return; + } + + const { height } = RollupMerkleTree; + const { nodes } = this; + + Array.from({ length: height }).forEach((ignored, level) => + Object.entries(nodes[level]).forEach((entry) => { + this.parent.setNode(BigInt(entry[0]), level, entry[1]); + }) + ); + + this.nodes = {} + } +} \ No newline at end of file diff --git a/packages/sequencer/src/protocol/production/tasks/providers/PreFilledStateService.ts b/packages/sequencer/src/state/prefilled/PreFilledStateService.ts similarity index 100% rename from packages/sequencer/src/protocol/production/tasks/providers/PreFilledStateService.ts rename to packages/sequencer/src/state/prefilled/PreFilledStateService.ts diff --git a/packages/sequencer/src/protocol/production/tasks/providers/PreFilledWitnessProvider.ts b/packages/sequencer/src/state/prefilled/PreFilledWitnessProvider.ts similarity index 100% rename from packages/sequencer/src/protocol/production/tasks/providers/PreFilledWitnessProvider.ts rename to packages/sequencer/src/state/prefilled/PreFilledWitnessProvider.ts diff --git a/packages/sequencer/src/protocol/production/execution/CachedStateService.ts b/packages/sequencer/src/state/state/CachedStateService.ts similarity index 91% rename from packages/sequencer/src/protocol/production/execution/CachedStateService.ts rename to packages/sequencer/src/state/state/CachedStateService.ts index 24d558aa..84d9097d 100644 --- a/packages/sequencer/src/protocol/production/execution/CachedStateService.ts +++ b/packages/sequencer/src/state/state/CachedStateService.ts @@ -1,8 +1,8 @@ import { Field } from "o1js"; -import { log } from "@proto-kit/common"; +import { log, noop } from "@proto-kit/common"; import { InMemoryStateService } from "@proto-kit/module"; -import { AsyncStateService } from "../state/AsyncStateService"; +import { AsyncStateService } from "../async/AsyncStateService"; const errors = { parentIsUndefined: () => new Error("Parent StateService is undefined"), @@ -28,6 +28,14 @@ export class CachedStateService } } + public commit(): void { + noop(); + } + + public openTransaction(): void { + noop(); + } + public async preloadKey(key: Field) { // Only preload it if it hasn't been preloaded previously if (this.parent !== undefined && this.get(key) === undefined) { diff --git a/packages/sequencer/src/protocol/production/execution/DummyStateService.ts b/packages/sequencer/src/state/state/DummyStateService.ts similarity index 100% rename from packages/sequencer/src/protocol/production/execution/DummyStateService.ts rename to packages/sequencer/src/state/state/DummyStateService.ts diff --git a/packages/sequencer/src/state/state/SyncCachedStateService.ts b/packages/sequencer/src/state/state/SyncCachedStateService.ts new file mode 100644 index 00000000..f3ec443a --- /dev/null +++ b/packages/sequencer/src/state/state/SyncCachedStateService.ts @@ -0,0 +1,45 @@ +import { InMemoryStateService } from "@proto-kit/module"; +import { Field } from "o1js"; +import { StateService } from "@proto-kit/protocol"; + +const errors = { + parentIsUndefined: () => new Error("Parent StateService is undefined"), +}; + +export class SyncCachedStateService + extends InMemoryStateService + implements StateService +{ + public constructor(private readonly parent: StateService | undefined) { + super(); + } + + public get(key: Field): Field[] | undefined { + return super.get(key) ?? this.parent?.get(key); + } + + private assertParentNotNull( + parent: StateService | undefined + ): asserts parent is StateService { + if (parent === undefined) { + throw errors.parentIsUndefined(); + } + } + + /** + * Merges all caches set() operation into the parent and + * resets this instance to the parent's state (by clearing the cache and + * defaulting to the parent) + */ + public async mergeIntoParent() { + const { parent, values } = this; + this.assertParentNotNull(parent); + + // Set all cached values on parent + Object.entries(values).map((value) => { + parent.set(Field(value[0]), value[1]); + }); + // Clear cache + this.values = {}; + } +} diff --git a/packages/sequencer/src/storage/MockStorageDependencyFactory.ts b/packages/sequencer/src/storage/MockStorageDependencyFactory.ts index 82118bfd..3fda9fc7 100644 --- a/packages/sequencer/src/storage/MockStorageDependencyFactory.ts +++ b/packages/sequencer/src/storage/MockStorageDependencyFactory.ts @@ -1,5 +1,4 @@ import { - AsyncMerkleTreeStore, InMemoryMerkleTreeStorage, StateServiceProvider, StateTransitionWitnessProviderReference, @@ -11,8 +10,9 @@ import { noop, } from "@proto-kit/common"; -import { AsyncStateService } from "../protocol/production/state/AsyncStateService"; -import { CachedStateService } from "../protocol/production/execution/CachedStateService"; +import { AsyncMerkleTreeStore } from "../state/async/AsyncMerkleTreeStore"; +import { CachedStateService } from "../state/state/CachedStateService"; +import { AsyncStateService } from "../state/async/AsyncStateService"; import { StorageDependencyFactory } from "./StorageDependencyFactory"; import { @@ -20,6 +20,11 @@ import { HistoricalBlockStorage, } from "./repositories/BlockStorage"; import { ComputedBlock } from "./model/Block"; +import { + HistoricalUnprovenBlockStorage, UnprovenBlockQueue, + UnprovenBlockStorage +} from "./repositories/UnprovenBlockStorage"; +import { UnprovenBlock } from "../protocol/production/unproven/TransactionExecutionService"; export class MockAsyncMerkleTreeStore implements AsyncMerkleTreeStore { private readonly store = new InMemoryMerkleTreeStorage(); @@ -64,6 +69,34 @@ class MockBlockStorage implements BlockStorage, HistoricalBlockStorage { } } +class MockUnprovenBlockStorage + implements UnprovenBlockStorage, HistoricalUnprovenBlockStorage +{ + private readonly blocks: UnprovenBlock[] = []; + + private cursor = 0; + + public async getBlockAt(height: number): Promise { + return this.blocks.at(height); + } + + public async getCurrentBlockHeight(): Promise { + return this.blocks.length; + } + + public async popNewBlocks(remove: boolean): Promise { + const slice = this.blocks.slice(this.cursor); + if (remove) { + this.cursor = this.blocks.length; + } + return slice; + } + + public async pushBlock(block: UnprovenBlock): Promise { + this.blocks.push(block); + } +} + @dependencyFactory() export class MockStorageDependencyFactory extends DependencyFactory @@ -71,6 +104,8 @@ export class MockStorageDependencyFactory { private readonly asyncService = new CachedStateService(undefined); + private readonly blockStorageQueue = new MockUnprovenBlockStorage(); + @dependency() public asyncMerkleStore(): AsyncMerkleTreeStore { return new MockAsyncMerkleTreeStore(); @@ -86,6 +121,16 @@ export class MockStorageDependencyFactory return new MockBlockStorage(); } + @dependency() + public unprovenBlockQueue(): UnprovenBlockQueue { + return this.blockStorageQueue; + } + + @dependency() + public unprovenBlockStorage(): UnprovenBlockStorage { + return this.blockStorageQueue; + } + @dependency() public stateServiceProvider(): StateServiceProvider { return new StateServiceProvider(this.asyncService); @@ -95,4 +140,9 @@ export class MockStorageDependencyFactory public stateTransitionWitnessProviderReference(): StateTransitionWitnessProviderReference { return new StateTransitionWitnessProviderReference(); } + + @dependency() + public unprovenStateService(): CachedStateService { + return new CachedStateService(this.asyncService); + } } diff --git a/packages/sequencer/src/storage/StorageDependencyFactory.ts b/packages/sequencer/src/storage/StorageDependencyFactory.ts index 15c5d205..15c5dbe2 100644 --- a/packages/sequencer/src/storage/StorageDependencyFactory.ts +++ b/packages/sequencer/src/storage/StorageDependencyFactory.ts @@ -1,20 +1,21 @@ import { inject } from "tsyringe"; -import { AsyncMerkleTreeStore } from "@proto-kit/protocol"; import { DependencyFactory, dependencyFactory } from "@proto-kit/common"; -import { AsyncStateService } from "../protocol/production/state/AsyncStateService"; +import { AsyncStateService } from "../state/async/AsyncStateService"; +import { AsyncMerkleTreeStore } from "../state/async/AsyncMerkleTreeStore"; import { Database } from "./Database"; import { BlockStorage } from "./repositories/BlockStorage"; +import { CachedStateService } from "../state/state/CachedStateService"; export interface StorageDependencyFactory { asyncStateService: () => AsyncStateService; asyncMerkleStore: () => AsyncMerkleTreeStore; + unprovenStateService: () => CachedStateService; blockStorage: () => BlockStorage; } @dependencyFactory() -// eslint-disable-next-line import/no-unused-modules export class DatabaseStorageDependencyFactory extends DependencyFactory { public constructor(@inject("Database") private readonly database: Database) { super(); diff --git a/packages/sequencer/src/storage/repositories/UnprovenBlockStorage.ts b/packages/sequencer/src/storage/repositories/UnprovenBlockStorage.ts new file mode 100644 index 00000000..2eff883b --- /dev/null +++ b/packages/sequencer/src/storage/repositories/UnprovenBlockStorage.ts @@ -0,0 +1,15 @@ +import { UnprovenBlock } from "../../protocol/production/unproven/TransactionExecutionService"; + +export interface UnprovenBlockQueue { + pushBlock: (block: UnprovenBlock) => Promise; + popNewBlocks: (remove: boolean) => Promise; +} + +export interface UnprovenBlockStorage { + getCurrentBlockHeight: () => Promise; + pushBlock: (block: UnprovenBlock) => Promise; +} + +export interface HistoricalUnprovenBlockStorage { + getBlockAt: (height: number) => Promise; +} diff --git a/packages/sequencer/test/integration/BlockProduction.test.ts b/packages/sequencer/test/integration/BlockProduction.test.ts index ef8e94b1..4de3c3dd 100644 --- a/packages/sequencer/test/integration/BlockProduction.test.ts +++ b/packages/sequencer/test/integration/BlockProduction.test.ts @@ -5,17 +5,15 @@ import "reflect-metadata"; // TODO this is actually a big issue // eslint-disable-next-line import/no-extraneous-dependencies import { AppChain } from "@proto-kit/sdk"; -import { - Fieldable, - Runtime, - MethodIdResolver, -} from "@proto-kit/module"; +import { Fieldable, Runtime, MethodIdResolver } from "@proto-kit/module"; import { AccountState, AccountStateModule, BlockProver, + Option, Path, Protocol, + StateTransition, StateTransitionProver, VanillaProtocol, } from "@proto-kit/protocol"; @@ -28,13 +26,15 @@ import { UnsignedTransaction } from "../../src/mempool/PendingTransaction"; import { Sequencer } from "../../src/sequencer/executor/Sequencer"; import { AsyncStateService, - BlockProducerModule, - ManualBlockTrigger, + BlockProducerModule, CachedMerkleTreeStore, + ManualBlockTrigger, MockAsyncMerkleTreeStore } from "../../src"; import { LocalTaskWorkerModule } from "../../src/worker/worker/LocalTaskWorkerModule"; import { Balance } from "./mocks/Balance"; import { NoopBaseLayer } from "../../src/protocol/baselayer/NoopBaseLayer"; +import { UnprovenProducerModule } from "../../src/protocol/production/unproven/UnprovenProducerModule"; +import { container } from "tsyringe"; describe("block production", () => { let runtime: Runtime<{ Balance: typeof Balance }>; @@ -43,6 +43,7 @@ describe("block production", () => { LocalTaskWorkerModule: typeof LocalTaskWorkerModule; BaseLayer: typeof NoopBaseLayer; BlockProducerModule: typeof BlockProducerModule; + UnprovenProducerModule: typeof UnprovenProducerModule; BlockTrigger: typeof ManualBlockTrigger; TaskQueue: typeof LocalTaskQueue; }>; @@ -53,6 +54,8 @@ describe("block production", () => { StateTransitionProver: typeof StateTransitionProver; }>; + let appChain: AppChain; + let blockTrigger: ManualBlockTrigger; let mempool: PrivateMempool; @@ -77,6 +80,7 @@ describe("block production", () => { LocalTaskWorkerModule, BaseLayer: NoopBaseLayer, BlockProducerModule, + UnprovenProducerModule, BlockTrigger: ManualBlockTrigger, TaskQueue: LocalTaskQueue, }, @@ -85,6 +89,7 @@ describe("block production", () => { BlockTrigger: {}, Mempool: {}, BlockProducerModule: {}, + UnprovenProducerModule: {}, LocalTaskWorkerModule: {}, BaseLayer: {}, TaskQueue: {}, @@ -93,7 +98,7 @@ describe("block production", () => { const protocolClass = VanillaProtocol.from( { AccountStateModule }, - { AccountStateModule: {}, StateTransitionProver: {}, BlockProver: {} } + { StateTransitionProver: {}, BlockProver: {}, AccountStateModule: {} } ); const app = AppChain.from({ @@ -103,8 +108,32 @@ describe("block production", () => { modules: {}, }); + app.configure({ + Sequencer: { + BlockTrigger: {}, + Mempool: {}, + BlockProducerModule: { + simulateProvers: true, + }, + UnprovenProducerModule: {}, + LocalTaskWorkerModule: {}, + BaseLayer: {}, + TaskQueue: {}, + }, + Runtime: { + Balance: {}, + }, + Protocol: { + AccountStateModule: {}, + BlockProver: {}, + StateTransitionProver: {}, + }, + }); + // Start AppChain - await app.start(); + await app.start(container.createChildContainer()); + + appChain = app; ({ runtime, sequencer, protocol } = app); @@ -247,7 +276,7 @@ describe("block production", () => { const numberTxs = 3; - it.only("should produce block with multiple transaction", async () => { + it("should produce block with multiple transaction", async () => { // eslint-disable-next-line jest/prefer-expect-assertions expect.assertions(5 + 2 * numberTxs); @@ -295,7 +324,7 @@ describe("block production", () => { ); }, 160_000); - it.each([ + it.skip.each([ [ "EKFZbsQfNiqjDiWGU7G3TVPauS3s9YgWgayMzjkEaDTEicsY9poM", "EKFdtp8D6mP3aFvCMRa75LPaUBn1QbmEs1YjTPXYLTNeqPYtnwy2", @@ -340,11 +369,13 @@ describe("block production", () => { const privateKey = PrivateKey.random(); + const field = Field(100); + mempool.add( createTransaction({ method: ["Balance", "lotOfSTs"], privateKey, - args: [], + args: [field], nonce: 0, }) ); @@ -372,7 +403,7 @@ describe("block production", () => { UInt64.from(100 * 10) ); - const pk2 = PublicKey.from({ x: Field(2), isOdd: Bool(false) }); + const pk2 = PublicKey.from({ x: field.add(Field(2)), isOdd: Bool(false) }); const balanceModule = runtime.resolve("Balance"); const balancesPath = Path.fromKey( balanceModule.balances.path!, diff --git a/packages/sequencer/test/integration/mocks/Balance.ts b/packages/sequencer/test/integration/mocks/Balance.ts index 463abf44..e1b68648 100644 --- a/packages/sequencer/test/integration/mocks/Balance.ts +++ b/packages/sequencer/test/integration/mocks/Balance.ts @@ -49,11 +49,14 @@ export class Balance extends RuntimeModule { @runtimeMethod() public addBalance(address: PublicKey, value: UInt64) { + const totalSupply = this.totalSupply.get(); + this.totalSupply.set(totalSupply.orElse(UInt64.zero).add(value)); + const balance = this.balances.get(address); log.provable.debug("Balance:", balance.isSome, balance.value); - const newBalance = balance.value.add(value); + const newBalance = balance.orElse(UInt64.zero).add(value); this.balances.set(address, newBalance); } @@ -73,10 +76,10 @@ export class Balance extends RuntimeModule { } @runtimeMethod() - public lotOfSTs() { + public lotOfSTs(randomArg: Field) { range(0, 10).forEach((index) => { // eslint-disable-next-line @typescript-eslint/no-magic-numbers - const pk = PublicKey.from({ x: Field(index % 5), isOdd: Bool(false) }); + const pk = PublicKey.from({ x: randomArg.add(Field(index % 5)), isOdd: Bool(false) }); const value = this.balances.get(pk); this.balances.set(pk, value.orElse(UInt64.zero).add(100)); const supply = this.totalSupply.get().orElse(UInt64.zero); diff --git a/packages/sequencer/test/merkle/CachedMerkleStore.test.ts b/packages/sequencer/test/merkle/CachedMerkleStore.test.ts index b02ff4db..ef579513 100644 --- a/packages/sequencer/test/merkle/CachedMerkleStore.test.ts +++ b/packages/sequencer/test/merkle/CachedMerkleStore.test.ts @@ -3,10 +3,7 @@ import { beforeEach, expect } from "@jest/globals"; import { Field } from "o1js"; import { MockAsyncMerkleTreeStore } from "../../src/storage/MockStorageDependencyFactory"; -import { - CachedMerkleTreeStore, - SyncCachedMerkleTreeStore, -} from "../../src/protocol/production/execution/CachedMerkleTreeStore"; +import { CachedMerkleTreeStore, SyncCachedMerkleTreeStore } from "../../src"; describe("cached merkle store", () => { const mainStore = new MockAsyncMerkleTreeStore(); diff --git a/packages/protocol/test/utils/MerkleTree.test.ts b/packages/sequencer/test/merkle/MerkleTree.test.ts similarity index 91% rename from packages/protocol/test/utils/MerkleTree.test.ts rename to packages/sequencer/test/merkle/MerkleTree.test.ts index 6d572767..c57520d7 100644 --- a/packages/protocol/test/utils/MerkleTree.test.ts +++ b/packages/sequencer/test/merkle/MerkleTree.test.ts @@ -1,8 +1,12 @@ -import { MockAsyncMerkleTreeStore } from "@proto-kit/module/test/state/MockAsyncMerkleStore"; -import { Field, Provable } from "o1js"; +import { Field } from "o1js"; -import { CachedMerkleTreeStore, RollupMerkleTree } from "../../src"; +import { RollupMerkleTree } from "@proto-kit/protocol"; import { log } from "@proto-kit/common"; +import { + CachedMerkleTreeStore, + MockAsyncMerkleTreeStore, + SyncCachedMerkleTreeStore +} from "../../src"; describe("merkle tree caching", () => { it("should cache, merge and cache again correctly", async () => { diff --git a/packages/sequencer/test/protocol/KeyExtraction.test.ts b/packages/sequencer/test/protocol/KeyExtraction.test.ts new file mode 100644 index 00000000..18eb8d23 --- /dev/null +++ b/packages/sequencer/test/protocol/KeyExtraction.test.ts @@ -0,0 +1,109 @@ +import { beforeAll, describe } from "@jest/globals"; +import { + InMemoryStateService, + Runtime, + runtimeMethod, + runtimeModule, + RuntimeModule, + state, +} from "@proto-kit/module"; +import { + NetworkState, + RuntimeMethodExecutionContext, + RuntimeTransaction, + State, + StateMap, +} from "@proto-kit/protocol"; +import { Field, PublicKey, UInt64 } from "o1js"; +import { TestingAppChain } from "@proto-kit/sdk"; + +import { CachedStateService } from "../../src"; +import { RuntimeMethodExecution } from "../../src/protocol/production/unproven/RuntimeMethodExecution"; + +@runtimeModule() +export class TestModule extends RuntimeModule<{}> { + @state() state1 = State.from(Field); + + @state() state2 = State.from(Field); + + @state() map = StateMap.from(Field, Field); + + @runtimeMethod() + public performAction(inputKey: Field) { + this.map.get(inputKey); + this.map.set(inputKey, Field(1)); + + const state1 = this.state1.get(); + const state2 = this.state2.get(); + + const compKey = state1.value.add(state2.value); + const value = this.map.get(compKey); + this.map.set(compKey, value.value.add(Field(10))); + } +} + +describe("test the correct key extraction for runtime methods", () => { + const stateService = new CachedStateService(undefined); + + let context: RuntimeMethodExecutionContext; + let execution: RuntimeMethodExecution; + let module: TestModule; + + beforeAll(async () => { + const appchain = TestingAppChain.fromRuntime({ + modules: { + TestModule, + }, + + config: { + TestModule: {}, + }, + }); + + await appchain.start(); + + module = appchain.runtime.resolve("TestModule"); + stateService.set(module.state1.path!, [Field(5)]); + stateService.set(module.state2.path!, [Field(10)]); + + context = appchain.runtime.dependencyContainer.resolve( + RuntimeMethodExecutionContext + ); + execution = new RuntimeMethodExecution( + appchain.runtime, + appchain.protocol, + context + ); + }); + + it("test if simulation is done correctly", async () => { + expect.assertions(1); + + const c = { + networkState: new NetworkState({ block: { height: UInt64.one } }), + + transaction: new RuntimeTransaction({ + sender: PublicKey.empty(), + nonce: UInt64.zero, + methodId: Field(0), + argsHash: Field(0), + }), + }; + + console.time("Simulating...") + + const sts = await execution.simulateMultiRound( + () => { + module.performAction(Field(5)); + }, + c, + new CachedStateService(stateService) + ); + + console.timeEnd("Simulating...") + + const path = module.map.getPath(Field(15)); + + expect(sts[4].path.toString()).toBe(path.toString()); + }); +});