feat(pass-style,marshal): ByteArray, a new binary Passable type

packages/marshal/src/deeplyFulfilled.js

@@ -62,6 +62,9 @@ export const deeplyFulfilled = async val => {
       const valPs = val.map(p => deeplyFulfilled(p));
       return E.when(Promise.all(valPs), vals => harden(vals));
     }
+    case 'byteArray': {
+      return val;
+    }
     case 'tagged': {
       const tag = getTag(val);
       return E.when(deeplyFulfilled(val.payload), payload =>

packages/marshal/src/encodePassable.js

@@ -10,7 +10,7 @@ import {
 } from '@endo/pass-style';
 
 /**
- * @import {CopyRecord, PassStyle, Passable, RemotableObject as Remotable} from '@endo/pass-style'
+ * @import {CopyRecord, PassStyle, Passable, RemotableObject as Remotable, ByteArray} from '@endo/pass-style'
  */
 
 import { b, q, Fail } from '@endo/errors';
@@ -462,6 +462,17 @@ const decodeLegacyArray = (encoded, decodePassable, skip = 0) => {
   return harden(elements);
 };
 
+/**
+ * @param {ByteArray} byteArray
+ * @param {(byteArray: ByteArray) => string} _encodePassable
+ * @returns {string}
+ */
+const encodeByteArray = (byteArray, _encodePassable) => {
+  // TODO implement
+  Fail`encodePassable(copyData) not yet implemented: ${byteArray}`;
+  return ''; // Just for the type
+};
+
 const encodeRecord = (record, encodeArray, encodePassable) => {
   const names = recordNames(record);
   const values = recordValues(record, names);
@@ -626,6 +637,9 @@ const makeInnerEncode = (encodeStringSuffix, encodeArray, options) => {
       case 'copyArray': {
         return encodeArray(passable, innerEncode);
       }
+      case 'byteArray': {
+        return encodeByteArray(passable, innerEncode);
+      }
       case 'copyRecord': {
         return encodeRecord(passable, encodeArray, innerEncode);
       }
@@ -870,6 +884,7 @@ export const passStylePrefixes = {
   tagged: ':',
   promise: '?',
   copyArray: '[^',
+  byteArray: '', // TODO pick a prefix
   boolean: 'b',
   number: 'f',
   bigint: 'np',

packages/marshal/src/encodeToCapData.js

@@ -194,6 +194,10 @@ export const makeEncodeToCapData = (encodeOptions = {}) => {
       case 'copyArray': {
         return passable.map(encodeToCapDataRecur);
       }
+      case 'byteArray': {
+        // TODO implement
+        throw Fail`marsal of byteArray not yet implemented: ${passable}`;
+      }
       case 'tagged': {
         return {
           [QCLASS]: 'tagged',

packages/marshal/src/encodeToSmallcaps.js

@@ -229,6 +229,10 @@ export const makeEncodeToSmallcaps = (encodeOptions = {}) => {
       case 'copyArray': {
         return passable.map(encodeToSmallcapsRecur);
       }
+      case 'byteArray': {
+        // TODO implement
+        throw Fail`marsal of byteArray not yet implemented: ${passable}`;
+      }
       case 'tagged': {
         return {
           '#tag': encodeToSmallcapsRecur(getTag(passable)),

packages/marshal/src/rankOrder.js

@@ -55,6 +55,8 @@ export const trivialComparator = (left, right) =>
 const passStyleRanks = /** @type {PassStyleRanksRecord} */ (
   fromEntries(
     entries(passStylePrefixes)
+      // TODO Until byteArray prefix is chosen
+      .filter(([_style, prefixes]) => prefixes.length >= 1)
       // Sort entries by ascending prefix.
       .sort(([_leftStyle, leftPrefixes], [_rightStyle, rightPrefixes]) => {
         return trivialComparator(leftPrefixes, rightPrefixes);
@@ -209,6 +211,26 @@ export const makeComparatorKit = (compareRemotables = (_x, _y) => 0) => {
         // If array X is a prefix of array Y, then X has an earlier rank than Y.
         return comparator(left.length, right.length);
       }
+      case 'byteArray': {
+        const leftArray = new Uint8Array(left.slice(0));
+        const rightArray = new Uint8Array(right.slice(0));
+        const byteLen = Math.min(left.byteLength, right.byteLength);
+        for (let i = 0; i < byteLen; i += 1) {
+          const leftByte = leftArray[i];
+          const rightByte = rightArray[i];
+          if (leftByte < rightByte) {
+            return -1;
+          }
+          if (leftByte > rightByte) {
+            return 1;
+          }
+        }
+        // If all corresponding bytes are the same,
+        // then according to their lengths.
+        // Thus, if the data of ByteArray X is a prefix of
+        // the data of ByteArray Y, then X is smaller than Y.
+        return comparator(left.byteLength, right.byteLength);
+      }
       case 'tagged': {
         // Lexicographic by `[Symbol.toStringTag]` then `.payload`.
         const labelComp = comparator(getTag(left), getTag(right));

packages/marshal/test/marshal-stringify.test.js

@@ -38,11 +38,13 @@ test('marshal stringify errors', t => {
     t.throws(() => stringify({}), {
       message: /Cannot pass non-frozen objects like .*. Use harden()/,
     });
-    // @ts-expect-error intentional error
+    // at-ts-ignore rather than at-expect-error because of disagreement
+    // @ts-ignore intentional error
     t.throws(() => stringify(harden(new Uint8Array(1))), {
       message: 'Cannot pass mutable typed arrays like "[Uint8Array]".',
     });
-    // @ts-expect-error intentional error
+    // at-ts-ignore rather than at-expect-error because of disagreement
+    // @ts-ignore intentional error
     t.throws(() => stringify(harden(new Int16Array(1))), {
       message: 'Cannot pass mutable typed arrays like "[Int16Array]".',
     });

packages/pass-style/src/byteArray.js

@@ -0,0 +1,55 @@
+import { X } from '@endo/errors';
+import { assertChecker } from './passStyle-helpers.js';
+
+const { getPrototypeOf, getOwnPropertyDescriptor } = Object;
+const { ownKeys, apply } = Reflect;
+
+// @ts-expect-error TODO How do I add it to the ArrayBuffer type?
+const AnImmutableArrayBuffer = new ArrayBuffer(0).transferToImmutable();
+/**
+ * As proposed, this will be the same as `ArrayBuffer.prototype`. As shimmed,
+ * this will be a hidden intrinsic that inherits from `ArrayBuffer.prototype`.
+ * Either way, get this in a way that we can trust it after lockdown, and
+ * require that all immutable ArrayBuffers directly inherit from it.
+ */
+const ImmutableArrayBufferPrototype = getPrototypeOf(AnImmutableArrayBuffer);
+
+// @ts-expect-error ok to implicitly assert the access is found
+const immutableGetter = getOwnPropertyDescriptor(
+  ImmutableArrayBufferPrototype,
+  'immutable',
+).get;
+
+/**
+ * @param {unknown} candidate
+ * @param {import('./types.js').Checker} [check]
+ * @returns {boolean}
+ */
+const canBeValid = (candidate, check = undefined) =>
+  (candidate instanceof ArrayBuffer &&
+    // @ts-expect-error TODO How do I add it to the ArrayBuffer type?
+    candidate.immutable) ||
+  (!!check && check(false, X`Immutable ArrayBuffer expected: ${candidate}`));
+
+/**
+ * @type {import('./internal-types.js').PassStyleHelper}
+ */
+export const ByteArrayHelper = harden({
+  styleName: 'byteArray',
+
+  canBeValid,
+
+  assertValid: (candidate, _passStyleOfRecur) => {
+    canBeValid(candidate, assertChecker);
+    getPrototypeOf(candidate) === ImmutableArrayBufferPrototype ||
+      assert.fail(X`Malformed ByteArray ${candidate}`, TypeError);
+    // @ts-expect-error assume immutableGetter was found
+    apply(immutableGetter, candidate, []) ||
+      assert.fail(X`Must be an immutable ArrayBuffer: ${candidate}`);
+    ownKeys(candidate).length === 0 ||
+      assert.fail(
+        X`ByteArrays must not have own properties: ${candidate}`,
+        TypeError,
+      );
+  },
+});

packages/pass-style/src/passStyleOf.js

@@ -7,6 +7,7 @@ import { X, Fail, q, annotateError, makeError } from '@endo/errors';
 import { isObject, isTypedArray, PASS_STYLE } from './passStyle-helpers.js';
 
 import { CopyArrayHelper } from './copyArray.js';
+import { ByteArrayHelper } from './byteArray.js';
 import { CopyRecordHelper } from './copyRecord.js';
 import { TaggedHelper } from './tagged.js';
 import {
@@ -43,6 +44,7 @@ const makeHelperTable = passStyleHelpers => {
   const HelperTable = {
     __proto__: null,
     copyArray: undefined,
+    byteArray: undefined,
     copyRecord: undefined,
     tagged: undefined,
     error: undefined,
@@ -216,6 +218,7 @@ export const passStyleOf =
   (globalThis && globalThis[PassStyleOfEndowmentSymbol]) ||
   makePassStyleOf([
     CopyArrayHelper,
+    ByteArrayHelper,
     CopyRecordHelper,
     TaggedHelper,
     ErrorHelper,

packages/pass-style/src/typeGuards.js

@@ -1,7 +1,9 @@
 import { Fail, q } from '@endo/errors';
 import { passStyleOf } from './passStyleOf.js';
 
-/** @import {CopyArray, CopyRecord, Passable, RemotableObject} from './types.js' */
+/**
+ * @import {CopyArray, CopyRecord, Passable, RemotableObject, ByteArray} from './types.js'
+ */
 
 /**
  * Check whether the argument is a pass-by-copy array, AKA a "copyArray"
@@ -13,6 +15,16 @@ import { passStyleOf } from './passStyleOf.js';
 const isCopyArray = arr => passStyleOf(arr) === 'copyArray';
 harden(isCopyArray);
 
+/**
+ * Check whether the argument is a pass-by-copy binary data, AKA a "byteArray"
+ * in @endo/marshal terms
+ *
+ * @param {Passable} arr
+ * @returns {arr is ByteArray}
+ */
+const isByteArray = arr => passStyleOf(arr) === 'byteArray';
+harden(isByteArray);
+
 /**
  * Check whether the argument is a pass-by-copy record, AKA a
  * "copyRecord" in @endo/marshal terms
@@ -47,6 +59,24 @@ const assertCopyArray = (array, optNameOfArray = 'Alleged array') => {
 harden(assertCopyArray);
 
 /**
+ * @callback AssertByteArray
+ * @param {Passable} array
+ * @param {string=} optNameOfArray
+ * @returns {asserts array is ByteArray}
+ */
+
+/** @type {AssertByteArray} */
+const assertByteArray = (array, optNameOfArray = 'Alleged byteArray') => {
+  const passStyle = passStyleOf(array);
+  passStyle === 'byteArray' ||
+    Fail`${q(
+      optNameOfArray,
+    )} ${array} must be a pass-by-copy binary data, not ${q(passStyle)}`;
+};
+harden(assertByteArray);
+
+/**
+ * @callback AssertRecord
  * @param {any} record
  * @param {string=} optNameOfRecord
  * @returns {asserts record is CopyRecord<any>}
@@ -80,8 +110,10 @@ harden(assertRemotable);
 export {
   assertRecord,
   assertCopyArray,
+  assertByteArray,
   assertRemotable,
   isRemotable,
   isRecord,
   isCopyArray,
+  isByteArray,
 };

packages/pass-style/src/types.d.ts

@@ -22,7 +22,11 @@ export type PrimitiveStyle =
   | 'string'
   | 'symbol';
 
-export type ContainerStyle = 'copyRecord' | 'copyArray' | 'tagged';
+export type ContainerStyle =
+  | 'copyRecord'
+  | 'copyArray'
+  | 'byteArray'
+  | 'tagged';
 
 export type PassStyle =
   | PrimitiveStyle
@@ -49,6 +53,7 @@ export type PassByCopy =
   | Primitive
   | Error
   | CopyArray
+  | ByteArray
   | CopyRecord
   | CopyTagged;
 
@@ -67,6 +72,7 @@ export type PassByRef =
  *     | 'string' | 'symbol').
  *   * Containers aggregate other Passables into
  *     * sequences as CopyArrays (PassStyle 'copyArray'), or
+ *     * sequences of 8-bit bytes (PassStyle 'byteArray'), or
  *     * string-keyed dictionaries as CopyRecords (PassStyle 'copyRecord'), or
  *     * higher-level types as CopyTaggeds (PassStyle 'tagged').
  *   * PassableCaps (PassStyle 'remotable' | 'promise') expose local values to
@@ -86,10 +92,12 @@ export type Passable<
 
 export type Container<PC extends PassableCap, E extends Error> =
   | CopyArrayI<PC, E>
+  | ByteArrayI
   | CopyRecordI<PC, E>
   | CopyTaggedI<PC, E>;
 interface CopyArrayI<PC extends PassableCap, E extends Error>
   extends CopyArray<Passable<PC, E>> {}
+interface ByteArrayI extends ByteArray {}
 interface CopyRecordI<PC extends PassableCap, E extends Error>
   extends CopyRecord<Passable<PC, E>> {}
 interface CopyTaggedI<PC extends PassableCap, E extends Error>
@@ -116,10 +124,9 @@ export type PassStyleOf = {
 /**
  * A Passable is PureData when its entire data structure is free of PassableCaps
  * (remotables and promises) and error objects.
- * PureData is an arbitrary composition of primitive values into CopyArray
- * and/or
- * CopyRecord and/or CopyTagged containers (or a single primitive value with no
- * container), and is fully pass-by-copy.
+ * PureData is an arbitrary composition of primitive values into CopyArray,
+ * ByteArray, CopyRecord, and/or CopyTagged containers
+ * (or a single primitive value with no container), and is fully pass-by-copy.
  *
  * This restriction assures absence of side effects and interleaving risks *given*
  * that none of the containers can be a Proxy instance.
@@ -156,6 +163,11 @@ export type PassableCap = Promise<any> | RemotableObject;
  */
 export type CopyArray<T extends Passable = any> = Array<T>;
 
+/**
+ * A `ByteArray` is a normal hardened immutable `ArrayBuffer`
+ */
+export type ByteArray = ArrayBuffer;
+
 /**
  * A Passable dictionary in which each key is a string and each value is Passable.
  */

packages/patterns/src/keys/checkKey.js

@@ -551,6 +551,9 @@ const checkKeyInternal = (val, check) => {
       // A copyArray is a key iff all its children are keys
       return val.every(checkIt);
     }
+    case 'byteArray': {
+      return true;
+    }
     case 'tagged': {
       const tag = getTag(val);
       switch (tag) {

packages/patterns/src/keys/compareKeys.js

@@ -147,6 +147,30 @@ export const compareKeys = (left, right) => {
       // @ts-expect-error narrowed
       return compareRank(left.length, right.length);
     }
+    case 'byteArray': {
+      // @ts-expect-error narrowed
+      const leftArray = new Uint8Array(left.slice(0));
+      // @ts-expect-error narrowed
+      const rightArray = new Uint8Array(right.slice(0));
+      // @ts-expect-error narrowed
+      const byteLen = Math.min(left.byteLength, right.byteLength);
+      for (let i = 0; i < byteLen; i += 1) {
+        const leftByte = leftArray[i];
+        const rightByte = rightArray[i];
+        if (leftByte < rightByte) {
+          return -1;
+        }
+        if (leftByte > rightByte) {
+          return 1;
+        }
+      }
+      // If all corresponding bytes are the same,
+      // then according to their lengths.
+      // Thus, if the data of ByteArray X is a prefix of
+      // the data of ByteArray Y, then X is smaller than Y.
+      // @ts-expect-error narrowed
+      return compareRank(left.byteLength, right.byteLength);
+    }
     case 'copyRecord': {
       // Pareto partial order comparison.
       // @ts-expect-error narrowed