Merge 96e33fb into 14c4457

packages/ssz/src/index.ts

@@ -21,10 +21,11 @@ export {BitArrayType} from "./type/bitArray";
 export {ByteArrayType} from "./type/byteArray";
 
 // Base type clases
-export {Type, ValueOf, JsonPath} from "./type/abstract";
+export {Type, ValueOf, JsonPath, ByteViews} from "./type/abstract";
 export {BasicType, isBasicType} from "./type/basic";
 export {CompositeType, CompositeTypeAny, CompositeView, CompositeViewDU, isCompositeType} from "./type/composite";
 export {TreeView} from "./view/abstract";
+export {ValueOfFields} from "./view/container";
 export {TreeViewDU} from "./viewDU/abstract";
 
 // Values

packages/ssz/src/type/arrayBasic.ts

@@ -104,12 +104,13 @@ export function tree_serializeToBytesArrayBasic<ElementType extends BasicType<un
   depth: number,
   output: ByteViews,
   offset: number,
-  node: Node
+  node: Node,
+  cachedNodes: Node[] | null = null
 ): number {
   const size = elementType.byteLength * length;
   const chunkCount = Math.ceil(size / 32);
 
-  const nodes = getNodesAtDepth(node, depth, 0, chunkCount);
+  const nodes = cachedNodes ?? getNodesAtDepth(node, depth, 0, chunkCount);
   packedNodeRootsToBytes(output.dataView, offset, size, nodes);
 
   return offset + size;

packages/ssz/src/type/arrayComposite.ts

@@ -151,9 +151,10 @@ export function tree_serializeToBytesArrayComposite<ElementType extends Composit
   depth: number,
   node: Node,
   output: ByteViews,
-  offset: number
+  offset: number,
+  cachedNodes: Node[] | null = null
 ): number {
-  const nodes = getNodesAtDepth(node, depth, 0, length);
+  const nodes = cachedNodes ?? getNodesAtDepth(node, depth, 0, length);
 
   // Variable Length
   // Indices contain offsets, which are indices deeper in the byte array

packages/ssz/src/type/container.ts

@@ -65,14 +65,14 @@ export class ContainerType<Fields extends Record<string, Type<unknown>>> extends
 
   // Precomputed data for faster serdes
   readonly fieldsEntries: FieldEntry<Fields>[];
+  /** End of fixed section of serialized Container */
+  readonly fixedEnd: number;
   protected readonly fieldsGindex: Record<keyof Fields, Gindex>;
   protected readonly jsonKeyToFieldName: Record<string, keyof Fields>;
   protected readonly isFixedLen: boolean[];
   protected readonly fieldRangesFixedLen: BytesRange[];
   /** Offsets position relative to start of serialized Container. Length may not equal field count. */
   protected readonly variableOffsetsPosition: number[];
-  /** End of fixed section of serialized Container */
-  protected readonly fixedEnd: number;
 
   /** Cached TreeView constuctor with custom prototype for this Type's properties */
   protected readonly TreeView: ContainerTreeViewTypeConstructor<Fields>;

packages/ssz/src/view/container.ts

@@ -19,6 +19,7 @@ export type ContainerTypeGeneric<Fields extends Record<string, Type<unknown>>> =
 > & {
   readonly fields: Fields;
   readonly fieldsEntries: FieldEntry<Fields>[];
+  readonly fixedEnd: number;
 };
 
 export type ValueOfFields<Fields extends Record<string, Type<unknown>>> = {[K in keyof Fields]: ValueOf<Fields[K]>};

packages/ssz/src/viewDU/abstract.ts

@@ -1,4 +1,4 @@
-import {CompositeType} from "../type/composite";
+import {ByteViews, CompositeType} from "../type/composite";
 import {TreeView} from "../view/abstract";
 
 /* eslint-disable @typescript-eslint/member-ordering  */
@@ -33,6 +33,13 @@ export abstract class TreeViewDU<T extends CompositeType<unknown, unknown, unkno
    */
   protected abstract clearCache(): void;
 
+  /*
+   * By default use type to serialize ViewDU.
+   */
+  serializeToBytes(output: ByteViews, offset: number): number {
+    return this.type.tree_serializeToBytes(output, offset, this.node);
+  }
+
   /**
    * Merkleize view and compute its hashTreeRoot.
    * Commits any pending changes before computing the root.
@@ -51,7 +58,10 @@ export abstract class TreeViewDU<T extends CompositeType<unknown, unknown, unkno
    */
   serialize(): Uint8Array {
     this.commit();
-    return super.serialize();
+    const output = new Uint8Array(this.type.tree_serializedSize(this.node));
+    const dataView = new DataView(output.buffer, output.byteOffset, output.byteLength);
+    this.serializeToBytes({uint8Array: output, dataView}, 0);
+    return output;
   }
 
   /**

packages/ssz/src/viewDU/arrayComposite.ts

@@ -216,7 +216,7 @@ export class ArrayCompositeTreeViewDU<
     }
   }
 
-  private populateAllNodes(): void {
+  protected populateAllNodes(): void {
     // If there's uncommited changes it may break.
     // this.length can be increased but this._rootNode doesn't have that item
     if (this.viewsChanged.size > 0) {

packages/ssz/src/viewDU/container.ts

@@ -1,5 +1,5 @@
 import {getNodeAtDepth, LeafNode, Node, setNodesAtDepth} from "@chainsafe/persistent-merkle-tree";
-import {Type} from "../type/abstract";
+import {ByteViews, Type} from "../type/abstract";
 import {BasicType, isBasicType} from "../type/basic";
 import {CompositeType, isCompositeType, CompositeTypeAny} from "../type/composite";
 import {ContainerTypeGeneric} from "../view/container";
@@ -114,6 +114,44 @@ class ContainerTreeViewDU<Fields extends Record<string, Type<unknown>>> extends
     // However preserving _SOME_ caches results in a very unpredictable experience.
     this.viewsChanged.clear();
   }
+
+  /**
+   * Same method to `type/container.ts` that call ViewDU.serializeToBytes() of internal fields.
+   */
+  serializeToBytes(output: ByteViews, offset: number): number {
+    this.commit();
+
+    let fixedIndex = offset;
+    let variableIndex = offset + this.type.fixedEnd;
+    for (let index = 0; index < this.type.fieldsEntries.length; index++) {
+      const {fieldType} = this.type.fieldsEntries[index];
+      let node = this.nodes[index];
+      if (node === undefined) {
+        node = getNodeAtDepth(this._rootNode, this.type.depth, index);
+        this.nodes[index] = node;
+      }
+      if (fieldType.fixedSize === null) {
+        // write offset
+        output.dataView.setUint32(fixedIndex, variableIndex - offset, true);
+        fixedIndex += 4;
+        // write serialized element to variable section
+        // basic types always have fixedSize
+        if (isCompositeType(fieldType)) {
+          const view = fieldType.getViewDU(node, this.caches[index]) as TreeViewDU<typeof fieldType>;
+          if (view.serializeToBytes !== undefined) {
+            variableIndex = view.serializeToBytes(output, variableIndex);
+          } else {
+            // some types don't define ViewDU as TreeViewDU, like the UnionType, in that case view.serializeToBytes = undefined
+            variableIndex = fieldType.tree_serializeToBytes(output, variableIndex, node);
+          }
+        }
+      } else {
+        fixedIndex = fieldType.tree_serializeToBytes(output, fixedIndex, node);
+      }
+    }
+
+    return variableIndex;
+  }
 }
 
 export function getContainerTreeViewDUClass<Fields extends Record<string, Type<unknown>>>(

packages/ssz/src/viewDU/listBasic.ts

@@ -6,10 +6,11 @@ import {
   treeZeroAfterIndex,
   zeroNode,
 } from "@chainsafe/persistent-merkle-tree";
-import {ValueOf} from "../type/abstract";
+import {ByteViews, ValueOf} from "../type/abstract";
 import {BasicType} from "../type/basic";
 import {ListBasicType} from "../view/listBasic";
 import {ArrayBasicTreeViewDU, ArrayBasicTreeViewDUCache} from "./arrayBasic";
+import {tree_serializeToBytesArrayBasic} from "../type/arrayBasic";
 
 export class ListBasicTreeViewDU<ElementType extends BasicType<unknown>> extends ArrayBasicTreeViewDU<ElementType> {
   constructor(readonly type: ListBasicType<ElementType>, protected _rootNode: Node, cache?: ArrayBasicTreeViewDUCache) {
@@ -78,4 +79,22 @@ export class ListBasicTreeViewDU<ElementType extends BasicType<unknown>> extends
     const newRootNode = this.type.tree_setChunksNode(rootNode, newChunksNode, newLength);
     return this.type.getViewDU(newRootNode) as this;
   }
+
+  /**
+   * Same method to `type/listBasic.ts` leveraging cached nodes.
+   */
+  serializeToBytes(output: ByteViews, offset: number): number {
+    this.commit();
+    const {nodes, nodesPopulated} = this.cache;
+    const chunksNode = this.type.tree_getChunksNode(this._rootNode);
+    return tree_serializeToBytesArrayBasic(
+      this.type.elementType,
+      this._length,
+      this.type.chunkDepth,
+      output,
+      offset,
+      chunksNode,
+      nodesPopulated ? nodes : null
+    );
+  }
 }

packages/ssz/src/viewDU/listComposite.ts

@@ -1,8 +1,9 @@
 import {Node, treeZeroAfterIndex} from "@chainsafe/persistent-merkle-tree";
-import {ValueOf} from "../type/abstract";
+import {ByteViews, ValueOf} from "../type/abstract";
 import {CompositeType, CompositeView, CompositeViewDU} from "../type/composite";
 import {ListCompositeType} from "../view/listComposite";
 import {ArrayCompositeTreeViewDU, ArrayCompositeTreeViewDUCache} from "./arrayComposite";
+import {tree_serializeToBytesArrayComposite} from "../type/arrayComposite";
 
 export class ListCompositeTreeViewDU<
   ElementType extends CompositeType<ValueOf<ElementType>, CompositeView<ElementType>, CompositeViewDU<ElementType>>
@@ -46,7 +47,7 @@ export class ListCompositeTreeViewDU<
     // Commit before getting rootNode to ensure all pending data is in the rootNode
     this.commit();
     const rootNode = this._rootNode;
-    const length = this.type.tree_getLength(rootNode);
+    const length = this._length;
 
     // All nodes beyond length are already zero
     // Array of length 2: [X,X,0,0], for index >= 1 no action needed
@@ -64,4 +65,22 @@ export class ListCompositeTreeViewDU<
 
     return this.type.getViewDU(newRootNode) as this;
   }
+
+  /**
+   * Same method to `type/listComposite.ts` leveraging cached nodes.
+   */
+  serializeToBytes(output: ByteViews, offset: number): number {
+    this.commit();
+    this.populateAllNodes();
+    const chunksNode = this.type.tree_getChunksNode(this._rootNode);
+    return tree_serializeToBytesArrayComposite(
+      this.type.elementType,
+      this._length,
+      this.type.chunkDepth,
+      chunksNode,
+      output,
+      offset,
+      this.nodes
+    );
+  }
 }

packages/ssz/test/spec/runValidTest.ts

@@ -5,6 +5,7 @@ import {fromHexString, toHexString} from "../../src/util/byteArray";
 import {CompositeType, isCompositeType} from "../../src/type/composite";
 import {isBasicType} from "../../src/type/basic";
 import {wrapErr} from "../utils/error";
+import {TreeViewDU} from "../../src";
 
 type ValidTestCaseData = {
   root: string;
@@ -169,6 +170,9 @@ export function runValidSszTest(type: Type<unknown>, testData: ValidTestCaseData
     const bytes = Buffer.from(copy(testDataSerialized));
     type.deserializeToViewDU(bytes);
     expect(toHexString(bytes)).to.equal(testDataSerializedHex, "type.deserializeToViewDU() mutated input");
+    if (viewDU instanceof TreeViewDU) {
+      assertBytes(viewDU.serialize(), "viewDU.serialize");
+    }
   }
 
   if (isBasicType(type)) {

packages/ssz/test/unit/byType/container/tree.test.ts

@@ -5,7 +5,9 @@ import {
   ContainerType,
   ListBasicType,
   ListCompositeType,
+  NoneType,
   toHexString,
+  UnionType,
   ValueOf,
 } from "../../../../src";
 import {uint64NumInfType, uint64NumType} from "../../../utils/primitiveTypes";
@@ -174,3 +176,45 @@ runViewTestMutation({
     },
   ],
 });
+
+// to test new the VietDU.serialize() implementation for different types
+const mixedContainer = new ContainerType({
+  // a basic type
+  a: uint64NumType,
+  // a list basic type
+  b: new ListBasicType(uint64NumType, 10),
+  // a list composite type
+  c: new ListCompositeType(new ContainerType({a: uint64NumInfType, b: uint64NumInfType}), 10),
+  // embedded container type
+  d: new ContainerType({a: uint64NumInfType}),
+  // a union type, cannot mutate through this test
+  e: new UnionType([new NoneType(), uint64NumInfType]),
+});
+
+runViewTestMutation({
+  type: mixedContainer,
+  mutations: [
+    {
+      id: "increase by 1",
+      valueBefore: {a: 10, b: [0, 1], c: [{a: 100, b: 101}], d: {a: 1000}, e: {selector: 1, value: 2000}},
+      //  View/ViewDU of Union is a value so we cannot mutate
+      valueAfter: {a: 11, b: [1, 2], c: [{a: 101, b: 102}], d: {a: 1001}, e: {selector: 1, value: 2000}},
+      fn: (tv) => {
+        tv.a += 1;
+        const b = tv.b;
+        for (let i = 0; i < b.length; i++) {
+          b.set(i, b.get(i) + 1);
+        }
+        const c = tv.c;
+        for (let i = 0; i < c.length; i++) {
+          const item = c.get(i);
+          item.a += 1;
+          item.b += 1;
+        }
+        tv.d.a += 1;
+        // does not affect anyway, leaving here to make it explicit
+        tv.e = {selector: 1, value: tv.e.value ?? 0 + 1};
+      },
+    },
+  ],
+});