From 75b3e38684f7230ae482fbe9d0411633b383a6ea Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 01:58:05 -0400
Subject: [PATCH 01/20] test(vats): Add tests for forthcoming chainStorage
 append messages

---
 packages/notifier/src/types.js              |  2 +-
 packages/vats/src/lib-chainStorage.js       |  4 +-
 packages/vats/test/test-lib-chainStorage.js | 69 +++++++++++++++++++++
 packages/vats/tools/storage-test-utils.js   | 30 ++++++++-
 4 files changed, 99 insertions(+), 6 deletions(-)

diff --git a/packages/notifier/src/types.js b/packages/notifier/src/types.js
index 4ddb1989189..4a52093b466 100644
--- a/packages/notifier/src/types.js
+++ b/packages/notifier/src/types.js
@@ -238,7 +238,7 @@
  * @property {(data: string) => void} setValue publishes some data
  * @property {() => ERef<VStorageKey>} getStoreKey get the
  * externally-reachable store key for this storage item
- * @property {(subPath: string) => StorageNode} makeChildNode
+ * @property {(subPath: string, options?: {sequence?: boolean}) => StorageNode} makeChildNode
  */
 
 /**
diff --git a/packages/vats/src/lib-chainStorage.js b/packages/vats/src/lib-chainStorage.js
index 84ff6fb52b3..fdccd772c77 100644
--- a/packages/vats/src/lib-chainStorage.js
+++ b/packages/vats/src/lib-chainStorage.js
@@ -55,8 +55,8 @@ export function makeChainStorageRoot(
           value: '',
         });
       },
-      /** @type {(name: string) => StorageNode} */
-      makeChildNode(name) {
+      /** @type {(name: string, childNodeOptions?: {sequence?: boolean}) => StorageNode} */
+      makeChildNode(name, childNodeOptions = {}) {
         assert.typeof(name, 'string');
         assertPathSegment(name);
         return makeChainStorageNode(`${path}.${name}`);
diff --git a/packages/vats/test/test-lib-chainStorage.js b/packages/vats/test/test-lib-chainStorage.js
index 6b2bf35aa8a..79baecf3646 100644
--- a/packages/vats/test/test-lib-chainStorage.js
+++ b/packages/vats/test/test-lib-chainStorage.js
@@ -171,3 +171,72 @@ test('makeChainStorageRoot', async t => {
     'child setValue message',
   );
 });
+
+test('makeChainStorageRoot sequence data', async t => {
+  const rootPath = 'root';
+  const { rootNode, messages } = makeFakeStorageKit(rootPath, {
+    sequence: true,
+  });
+
+  // @ts-expect-error
+  t.throws(() => rootNode.setValue([]), undefined, 'array value is rejected');
+
+  rootNode.setValue('foo');
+  t.deepEqual(
+    messages.slice(-1),
+    [{ key: rootPath, method: 'append', value: 'foo' }],
+    'root setValue message',
+  );
+  rootNode.setValue('bar');
+  t.deepEqual(
+    messages.slice(-1),
+    [{ key: rootPath, method: 'append', value: 'bar' }],
+    'second setValue message',
+  );
+
+  // Child nodes inherit configuration unless overridden.
+  let childNode = rootNode.makeChildNode('child');
+  const childPath = `${rootPath}.child`;
+  let deepNode = childNode.makeChildNode('grandchild');
+  const deepPath = `${childPath}.grandchild`;
+  childNode.setValue('foo');
+  t.deepEqual(
+    messages.slice(-1),
+    [{ key: childPath, method: 'append', value: 'foo' }],
+    'auto-sequence child setValue message',
+  );
+  deepNode.setValue('foo');
+  t.deepEqual(
+    messages.slice(-1),
+    [{ key: deepPath, method: 'append', value: 'foo' }],
+    'auto-sequence grandchild setValue message',
+  );
+  deepNode = childNode.makeChildNode('grandchild', { sequence: false });
+  deepNode.setValue('bar');
+  t.deepEqual(
+    messages.slice(-1),
+    [{ key: deepPath, method: 'set', value: 'bar' }],
+    'manual-single grandchild setValue message',
+  );
+  childNode = rootNode.makeChildNode('child', { sequence: false });
+  childNode.setValue('bar');
+  t.deepEqual(
+    messages.slice(-1),
+    [{ key: childPath, method: 'set', value: 'bar' }],
+    'manual-single child setValue message',
+  );
+  deepNode = childNode.makeChildNode('grandchild');
+  deepNode.setValue('baz');
+  t.deepEqual(
+    messages.slice(-1),
+    [{ key: deepPath, method: 'set', value: 'baz' }],
+    'auto-single grandchild setValue message',
+  );
+  deepNode = childNode.makeChildNode('grandchild', { sequence: true });
+  deepNode.setValue('qux');
+  t.deepEqual(
+    messages.slice(-1),
+    [{ key: deepPath, method: 'append', value: 'qux' }],
+    'manual-sequence grandchild setValue message',
+  );
+});
diff --git a/packages/vats/tools/storage-test-utils.js b/packages/vats/tools/storage-test-utils.js
index 8d24a29e2bd..b9d3d246da6 100644
--- a/packages/vats/tools/storage-test-utils.js
+++ b/packages/vats/tools/storage-test-utils.js
@@ -7,9 +7,10 @@ import { makeChainStorageRoot } from '../src/lib-chainStorage.js';
  * and exposes both the map and the sequence of received messages.
  *
  * @param {string} rootPath
+ * @param {object} [rootOptions]
  */
-export const makeFakeStorageKit = rootPath => {
-  /** @type {Map<string, string>} */
+export const makeFakeStorageKit = (rootPath, rootOptions) => {
+  /** @type {Map<string, any>} */
   const data = new Map();
   /** @type {import('../src/lib-chainStorage.js').StorageMessage[]} */
   const messages = [];
@@ -31,6 +32,24 @@ export const makeFakeStorageKit = rootPath => {
           data.delete(message.key);
         }
         break;
+      case 'append':
+        if ('value' in message) {
+          let sequence = data.get(message.key);
+          if (!Array.isArray(sequence)) {
+            if (sequence === undefined) {
+              // Initialize an empty collection.
+              sequence = [];
+            } else {
+              // Wrap a previous single value in a collection.
+              sequence = [sequence];
+            }
+            data.set(message.key, sequence);
+          }
+          sequence.push(message.value);
+        } else {
+          throw new Error(`attempt to append with no value`);
+        }
+        break;
       case 'size':
         // Intentionally incorrect because it counts non-child descendants,
         // but nevertheless supports a "has children" test.
@@ -40,7 +59,12 @@ export const makeFakeStorageKit = rootPath => {
         throw new Error(`unsupported method: ${message.method}`);
     }
   };
-  const rootNode = makeChainStorageRoot(toStorage, 'swingset', rootPath);
+  const rootNode = makeChainStorageRoot(
+    toStorage,
+    'swingset',
+    rootPath,
+    rootOptions,
+  );
   return { rootNode, data, messages };
 };
 harden(makeFakeStorageKit);

From 461204af30c5437a072d17a1703f3ec02395721b Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 02:03:04 -0400
Subject: [PATCH 02/20] feat(vats): Add support for configuring chainStorage
 nodes as sequences

---
 golang/cosmos/x/vstorage/keeper/keeper.go | 40 +++++++++++++++++++++++
 golang/cosmos/x/vstorage/vstorage.go      |  8 +++++
 packages/vats/src/lib-chainStorage.js     | 17 +++++++---
 packages/vats/src/vat-chainStorage.js     | 15 +++++++--
 4 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/golang/cosmos/x/vstorage/keeper/keeper.go b/golang/cosmos/x/vstorage/keeper/keeper.go
index d9858aa2dde..b9e7e3c546b 100644
--- a/golang/cosmos/x/vstorage/keeper/keeper.go
+++ b/golang/cosmos/x/vstorage/keeper/keeper.go
@@ -2,6 +2,8 @@ package keeper
 
 import (
 	"bytes"
+	"encoding/json"
+	"strconv"
 	"strings"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -11,6 +13,19 @@ import (
 	"github.com/Agoric/agoric-sdk/golang/cosmos/x/vstorage/types"
 )
 
+// StreamCell is an envelope representing a sequence of values written at a path in a single block.
+// It is persisted to storage as a { "height": "<digits>", "values": ["...", ...] } JSON text
+// that off-chain consumers rely upon.
+type StreamCell struct {
+	Height string `json:"height"`
+	// XXX Should Values be []string or []interface{}?
+	// The latter would remove a layer of JSON encoding (e.g., `[{…}]` rather than `["{…}"]`,
+	// but would add a requirement exclusive to AppendStorageValueAndNotify that its input be JSON.
+	// On the other hand, we could always extend this format in the future to include an indication
+	// that values are subject to a different encoding, e.g. `"valueEncoding":"base64"`.
+	Values []string `json:"values"`
+}
+
 // Keeper maintains the link to data storage and exposes getter/setter methods
 // for the various parts of the state machine
 type Keeper struct {
@@ -127,6 +142,31 @@ func (k Keeper) SetStorageAndNotify(ctx sdk.Context, path, value string) {
 	)
 }
 
+func (k Keeper) AppendStorageValueAndNotify(ctx sdk.Context, path, value string) error {
+	height := strconv.FormatInt(ctx.BlockHeight(), 10)
+
+	// Preserve correctly-formatted data within the current block,
+	// otherwise initialize a blank cell.
+	currentData := k.GetData(ctx, path)
+	var cell StreamCell
+	_ = json.Unmarshal([]byte(currentData), &cell)
+	if cell.Height != height {
+		cell.Height = height
+		cell.Values = make([]string, 0, 1)
+	}
+
+	// Append the new value.
+	cell.Values = append(cell.Values, value)
+
+	// Perform the write.
+	bz, err := json.Marshal(cell)
+	if err != nil {
+		return err
+	}
+	k.SetStorageAndNotify(ctx, path, string(bz))
+	return nil
+}
+
 func componentsToPath(components []string) string {
 	return strings.Join(components, types.PathSeparator)
 }
diff --git a/golang/cosmos/x/vstorage/vstorage.go b/golang/cosmos/x/vstorage/vstorage.go
index 84ea1fdb0e2..afdfd6998fe 100644
--- a/golang/cosmos/x/vstorage/vstorage.go
+++ b/golang/cosmos/x/vstorage/vstorage.go
@@ -61,7 +61,15 @@ func (sh vstorageHandler) Receive(cctx *vm.ControllerContext, str string) (ret s
 		keeper.SetStorageAndNotify(cctx.Context, msg.Path, msg.Value)
 		return "true", nil
 
+	case "append":
+		err = keeper.AppendStorageValueAndNotify(cctx.Context, msg.Path, msg.Value)
+		if err != nil {
+			return "", err
+		}
+		return "true", nil
+
 	case "get":
+		// Note that "get" does not (currently) unwrap a StreamCell.
 		value := keeper.GetData(cctx.Context, msg.Path)
 		if value == "" {
 			return "null", nil
diff --git a/packages/vats/src/lib-chainStorage.js b/packages/vats/src/lib-chainStorage.js
index fdccd772c77..a7a60bd3082 100644
--- a/packages/vats/src/lib-chainStorage.js
+++ b/packages/vats/src/lib-chainStorage.js
@@ -32,11 +32,14 @@ harden(assertPathSegment);
  * @param {(message: StorageMessage) => any} handleStorageMessage a function for sending a storageMessage object to the storage implementation (cf. golang/cosmos/x/vstorage/vstorage.go)
  * @param {'swingset'} storeName currently limited to "swingset"
  * @param {string} rootPath
+ * @param {object} [rootOptions]
+ * @param {boolean} [rootOptions.sequence] employ a wrapping structure that preserves each value set within a single block, and default child nodes to do the same
  */
 export function makeChainStorageRoot(
   handleStorageMessage,
   storeName,
   rootPath,
+  rootOptions = {},
 ) {
   assert.equal(
     storeName,
@@ -45,7 +48,8 @@ export function makeChainStorageRoot(
   );
   assert.typeof(rootPath, 'string');
 
-  function makeChainStorageNode(path) {
+  function makeChainStorageNode(path, options = {}) {
+    const { sequence = false } = options;
     const node = {
       /** @type {() => VStorageKey} */
       getStoreKey() {
@@ -59,12 +63,17 @@ export function makeChainStorageRoot(
       makeChildNode(name, childNodeOptions = {}) {
         assert.typeof(name, 'string');
         assertPathSegment(name);
-        return makeChainStorageNode(`${path}.${name}`);
+        const mergedOptions = { sequence, ...childNodeOptions };
+        return makeChainStorageNode(`${path}.${name}`, mergedOptions);
       },
       /** @type {(value: string) => void} */
       setValue(value) {
         assert.typeof(value, 'string');
-        handleStorageMessage({ key: path, method: 'set', value });
+        handleStorageMessage({
+          key: path,
+          method: sequence ? 'append' : 'set',
+          value,
+        });
       },
       // Possible extensions:
       // * getValue()
@@ -77,7 +86,7 @@ export function makeChainStorageRoot(
     return Far('chainStorageNode', node);
   }
 
-  const rootNode = makeChainStorageNode(rootPath);
+  const rootNode = makeChainStorageNode(rootPath, rootOptions);
   return rootNode;
 }
 
diff --git a/packages/vats/src/vat-chainStorage.js b/packages/vats/src/vat-chainStorage.js
index 9e020cf0e71..9020eda9606 100644
--- a/packages/vats/src/vat-chainStorage.js
+++ b/packages/vats/src/vat-chainStorage.js
@@ -7,12 +7,23 @@ export function buildRootObject(_vatPowers) {
    * @param {ERef<BridgeManager>} bridgeManager
    * @param {string} bridgeId
    * @param {string} rootPath must be unique (caller responsibility to ensure)
+   * @param {object} [options]
    */
-  function makeBridgedChainStorageRoot(bridgeManager, bridgeId, rootPath) {
+  function makeBridgedChainStorageRoot(
+    bridgeManager,
+    bridgeId,
+    rootPath,
+    options,
+  ) {
     // Note that the uniqueness of rootPath is not validated here,
     // and is instead the responsibility of callers.
     const toStorage = message => E(bridgeManager).toBridge(bridgeId, message);
-    const rootNode = makeChainStorageRoot(toStorage, 'swingset', rootPath);
+    const rootNode = makeChainStorageRoot(
+      toStorage,
+      'swingset',
+      rootPath,
+      options,
+    );
     return rootNode;
   }
 

From 35db0daed7f8315222fa87cbf9c50e4e2ee8d225 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 05:39:23 -0400
Subject: [PATCH 03/20] feat(casting): Update to consume stream cells

Fixes #5366
---
 packages/casting/src/follower-cosmjs.js  | 41 ++++++++----
 packages/casting/test/fake-rpc-server.js | 37 +++++++++--
 packages/casting/test/test-mvp.js        | 81 +++++++++++++++---------
 3 files changed, 111 insertions(+), 48 deletions(-)

diff --git a/packages/casting/src/follower-cosmjs.js b/packages/casting/src/follower-cosmjs.js
index 0a95a278f6e..66f40b34036 100644
--- a/packages/casting/src/follower-cosmjs.js
+++ b/packages/casting/src/follower-cosmjs.js
@@ -305,7 +305,7 @@ export const makeCosmjsFollower = (
      * @param {import('./types').CastingChange} allegedChange
      */
     const tryQueryAndUpdate = async allegedChange => {
-      const committer = prepareUpdateInOrder();
+      let committer = prepareUpdateInOrder();
 
       // Make an unproven query if we have no alleged value.
       const { values: allegedValues, blockHeight: allegedBlockHeight } =
@@ -334,18 +334,36 @@ export const makeCosmjsFollower = (
         }
       }
       lastBuf = buf;
-      const data = decode(buf);
-      if (!unserializer) {
-        /** @type {T} */
-        const value = data;
-        committer.commit({ value });
-        return;
+      let streamCell = decode(buf);
+      // Upgrade a naked value to a JSON stream cell if necessary.
+      if (!streamCell.height || !streamCell.values) {
+        streamCell = { values: [JSON.stringify(streamCell)] };
       }
-      const value = await E(unserializer).unserialize(data);
-      if (!committer.isValid()) {
-        return;
+      for (let i = 0; i < streamCell.values.length; i += 1) {
+        const data = JSON.parse(streamCell.values[i]);
+        const last = i + 1 === streamCell.values.length;
+        if (!unserializer) {
+          /** @type {T} */
+          const value = data;
+          committer.commit({ value });
+          if (!last) {
+            committer = prepareUpdateInOrder();
+          }
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+        // eslint-disable-next-line no-await-in-loop,@jessie.js/no-nested-await
+        const value = await E(unserializer).unserialize(data);
+        if (!committer.isValid()) {
+          // QUESTION: How would we get here, and what is the proper handling?
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+        committer.commit({ value });
+        if (!last) {
+          committer = prepareUpdateInOrder();
+        }
       }
-      committer.commit({ value });
     };
 
     const changeFollower = E(leader).watchCasting(castingSpecP);
@@ -362,6 +380,7 @@ export const makeCosmjsFollower = (
           return;
         }
         harden(allegedChange);
+        // eslint-disable-next-line @jessie.js/no-nested-await
         await queryAndUpdateOnce(allegedChange);
       }
     };
diff --git a/packages/casting/test/fake-rpc-server.js b/packages/casting/test/fake-rpc-server.js
index 0db8d1ba8ba..118c3646ce5 100644
--- a/packages/casting/test/fake-rpc-server.js
+++ b/packages/casting/test/fake-rpc-server.js
@@ -64,10 +64,18 @@ const fakeStatusResult = {
   },
 };
 
-export const startFakeServer = (t, fakeValues, marshaller = makeMarshal()) => {
+/**
+ * @param {Assertions} t
+ * @param {Array<{any}>} fakeValues
+ * @param {object} [options]
+ * @param {Marshaller} [options.marshaller]
+ * @param {number} [options.batchSize] count of stream-cell results per response, or 0/absent to return lone naked values
+ */
+export const startFakeServer = (t, fakeValues, options = {}) => {
   const { log = console.log } = t;
   lastPort += 1;
   const PORT = lastPort;
+  const { marshaller = makeMarshal(), batchSize = 0 } = options;
   return new Promise(resolve => {
     log('starting http server on port', PORT);
     const app = express();
@@ -97,6 +105,8 @@ export const startFakeServer = (t, fakeValues, marshaller = makeMarshal()) => {
       buf.set(ascii, dataPrefix.length);
       return toBase64(buf);
     };
+    let height = 74863;
+    let responseValueBase64;
     app.post('/tendermint-rpc', (req, res) => {
       log('received', req.path, req.body, req.params);
       const reply = result => {
@@ -114,10 +124,23 @@ export const startFakeServer = (t, fakeValues, marshaller = makeMarshal()) => {
           break;
         }
         case 'abci_query': {
-          const value =
-            fakeValues.length === 0
-              ? null
-              : encode(marshaller.serialize(fakeValues.shift()));
+          height += 2;
+          const values = fakeValues.splice(0, Math.max(1, batchSize));
+          if (values.length > 0) {
+            if (batchSize > 0) {
+              // Return a JSON stream cell.
+              const serializedValues = values.map(val =>
+                JSON.stringify(marshaller.serialize(val)),
+              );
+              responseValueBase64 = encode({
+                height: String(height - 1),
+                values: serializedValues,
+              });
+            } else {
+              // Return a single naked value.
+              responseValueBase64 = encode(marshaller.serialize(values[0]));
+            }
+          }
           const result = {
             response: {
               code: 0,
@@ -127,9 +150,9 @@ export const startFakeServer = (t, fakeValues, marshaller = makeMarshal()) => {
               key: Buffer.from(
                 'swingset/data:mailbox.agoric1foobarbaz',
               ).toString('base64'),
-              value,
+              value: responseValueBase64,
               proofOps: null,
-              height: '74863',
+              height: String(height),
               codespace: '',
             },
           };
diff --git a/packages/casting/test/test-mvp.js b/packages/casting/test/test-mvp.js
index d3a3578030e..e7c7cccef86 100644
--- a/packages/casting/test/test-mvp.js
+++ b/packages/casting/test/test-mvp.js
@@ -12,39 +12,60 @@ import {
 import { delay } from '../src/defaults.js';
 import { startFakeServer } from './fake-rpc-server.js';
 
-test('happy path', async t => {
-  const expected = ['latest', 'later', 'done'];
-  t.plan(expected.length);
-  const PORT = await t.context.startServer(t, [...expected]);
-  /** @type {import('../src/types.js').LeaderOptions} */
-  const lo = {
-    retryCallback: null, // fail fast, no retries
-    keepPolling: () => delay(200).then(() => true), // poll really quickly
-    jitter: null, // no jitter
-  };
-  /** @type {import('../src/types.js').FollowerOptions} */
-  const so = {
-    proof: 'none',
-  };
+// TODO: Replace with test.macro({title, exec}).
+const testHappyPath = (label, ...input) => {
+  // eslint-disable-next-line no-shadow
+  const title = label => `happy path ${label}`;
+  const makeExec =
+    ({ fakeValues, options }) =>
+    async t => {
+      const expected = fakeValues;
+      t.plan(expected.length);
+      const PORT = await t.context.startFakeServer(t, [...expected], options);
+      /** @type {import('../src/types.js').LeaderOptions} */
+      const lo = {
+        retryCallback: null, // fail fast, no retries
+        keepPolling: () => delay(200).then(() => true), // poll really quickly
+        jitter: null, // no jitter
+      };
+      /** @type {import('../src/types.js').FollowerOptions} */
+      const so = {
+        proof: 'none',
+      };
 
-  // The rest of this test is taken almost verbatim from the README.md, with
-  // some minor modifications (testLeaderOptions and deepEqual).
-  const leader = makeLeader(`http://localhost:${PORT}/network-config`, lo);
-  const castingSpec = makeCastingSpec(':mailbox.agoric1foobarbaz');
-  const follower = await makeFollower(castingSpec, leader, so);
-  for await (const { value } of iterateLatest(follower)) {
-    t.log(`here's a mailbox value`, value);
+      // The rest of this test is taken almost verbatim from the README.md, with
+      // some minor modifications (testLeaderOptions and deepEqual).
+      const leader = makeLeader(`http://localhost:${PORT}/network-config`, lo);
+      const castingSpec = makeCastingSpec(':mailbox.agoric1foobarbaz');
+      const follower = await makeFollower(castingSpec, leader, so);
+      for await (const { value } of iterateLatest(follower)) {
+        t.log(`here's a mailbox value`, value);
 
-    // The rest here is to drive the test.
-    t.deepEqual(value, expected.shift());
-    if (expected.length === 0) {
-      break;
-    }
-  }
+        // The rest here is to drive the test.
+        t.deepEqual(value, expected.shift());
+        if (expected.length === 0) {
+          break;
+        }
+      }
+    };
+  test(title(label), makeExec(...input));
+};
+
+testHappyPath('naked values', {
+  fakeValues: ['latest', 'later', 'done'],
+  options: {},
+});
+testHappyPath('batchSize=1', {
+  fakeValues: ['latest', 'later', 'done'],
+  options: { batchSize: 1 },
+});
+testHappyPath('batchSize=2', {
+  fakeValues: ['latest', 'later', 'done'],
+  options: { batchSize: 2 },
 });
 
 test('bad network config', async t => {
-  const PORT = await t.context.startServer(t, []);
+  const PORT = await t.context.startFakeServer(t, []);
   await t.throwsAsync(
     () =>
       makeLeader(`http://localhost:${PORT}/bad-network-config`, {
@@ -58,7 +79,7 @@ test('bad network config', async t => {
 });
 
 test('missing rpc server', async t => {
-  const PORT = await t.context.startServer(t, []);
+  const PORT = await t.context.startFakeServer(t, []);
   await t.throwsAsync(
     () =>
       makeLeader(`http://localhost:${PORT}/missing-network-config`, {
@@ -83,7 +104,7 @@ test('unrecognized proof', async t => {
 
 test.before(t => {
   t.context.cleanups = [];
-  t.context.startServer = startFakeServer;
+  t.context.startFakeServer = startFakeServer;
 });
 
 test.after(t => {

From 52de53540fbd80be0890a7e930c5f2dfc2d6bd7c Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 05:39:23 -0400
Subject: [PATCH 04/20] feat(scripts): Update get-flattened-publication to
 consume stream cells

---
 scripts/get-flattened-publication.sh | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/scripts/get-flattened-publication.sh b/scripts/get-flattened-publication.sh
index a357db7f564..6f6060b3168 100755
--- a/scripts/get-flattened-publication.sh
+++ b/scripts/get-flattened-publication.sh
@@ -60,17 +60,26 @@ curl -sS "$URL_PREFIX" --request POST --header 'Content-Type: application/json'
 )" | \
 # Decode, simplify, flatten, and compact the output.
 jq -c '
-  # Capture block height.
-  .result.response.height? as $height |
+  # Capture response block height.
+  .result.response.height? as $responseHeight |
 
   # Decode `value` as base64, then decode that as JSON.
   .result.response.value | @base64d | fromjson |
 
-  # Decode `value` as JSON, capture `slots`, then decode `body` as JSON.
-  .value | fromjson | .slots as $slots | .body | fromjson |
+  # Decode `value` as JSON.
+  .value | fromjson |
 
-  # Add block height.
-  (.blockHeight |= $height) |
+  # Upgrade a naked value to a stream cell if necessary.
+  if has("height") and has("values") then . else { values: [ . | tojson ] } end |
+
+  # Capture data block height.
+  .height as $dataHeight |
+
+  # Flatten each value independently.
+  .values[] | fromjson |
+
+  # Capture `slots`, then decode `body` as JSON.
+  .slots as $slots | .body | fromjson |
 
   # Replace select capdata.
   walk(
@@ -92,5 +101,9 @@ jq -c '
   ) |
 
   # Flatten the resulting structure, joining deep member names with "-".
-  [ paths(scalars) as $path | { key: $path | join("-"), value: getpath($path) } ] | from_entries
+  [ paths(scalars) as $path | { key: $path | join("-"), value: getpath($path) } ] | from_entries |
+
+  # Add block height information.
+  (.dataBlockHeight |= $dataHeight) |
+  (.blockHeight |= $responseHeight)
 '

From 6bf38789bc03b644666880c8d999d29deb45f8e3 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 12:54:49 -0400
Subject: [PATCH 05/20] chore(vats): Update typedef to support new
 StorageMessage method "append"

---
 packages/vats/src/lib-chainStorage.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/vats/src/lib-chainStorage.js b/packages/vats/src/lib-chainStorage.js
index a7a60bd3082..ac3f7976236 100644
--- a/packages/vats/src/lib-chainStorage.js
+++ b/packages/vats/src/lib-chainStorage.js
@@ -22,7 +22,7 @@ harden(assertPathSegment);
 /**
  * Must match the switch in vstorage.go using `vstorageMessage` type
  *
- * @typedef {'get' | 'getStoreKey' | 'set' | 'has' |'entries' | 'values' |'size' } StorageMessageMethod
+ * @typedef {'get' | 'getStoreKey' | 'set' | 'append' | 'has' |'entries' | 'values' |'size' } StorageMessageMethod
  * @typedef {{key: string, method: StorageMessageMethod, value: string}} StorageMessage
  */
 

From 09e165cf3cf93775b4e978b578a4ce7699239030 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 14:01:00 -0400
Subject: [PATCH 06/20] chore(casting): Fix parameter types for fake RPC server

---
 packages/casting/test/fake-rpc-server.js | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/packages/casting/test/fake-rpc-server.js b/packages/casting/test/fake-rpc-server.js
index 118c3646ce5..d674bcee572 100644
--- a/packages/casting/test/fake-rpc-server.js
+++ b/packages/casting/test/fake-rpc-server.js
@@ -64,8 +64,9 @@ const fakeStatusResult = {
   },
 };
 
+/** @typedef {Partial<import('ava').ExecutionContext<{cleanups: Array<() => void>}>> & {context}} fakeServerTestContext */
 /**
- * @param {Assertions} t
+ * @param {fakeServerTestContext} t
  * @param {Array<{any}>} fakeValues
  * @param {object} [options]
  * @param {Marshaller} [options.marshaller]
@@ -249,10 +250,12 @@ export const develop = async () => {
       unserialize({ body: jsonMarshalled, slots: [] }),
     ),
   );
-  const mockT = {
-    log: console.log,
-    context: { cleanups: [] },
-  };
+  const mockT = /** @type {fakeServerTestContext} */ (
+    /** @type {unknown} */ ({
+      log: console.log,
+      context: { cleanups: [] },
+    })
+  );
   const PORT = await startFakeServer(mockT, [...fakeValues]);
   console.log(
     `Try this in another terminal:

From a427c591e8693b3e33e84462f5af5b7e8d623892 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 14:53:54 -0400
Subject: [PATCH 07/20] feat(vats)!: Switch the "published" chain storage
 subtree to stream cells

Fixes #5508
---
 packages/vats/src/core/chain-behaviors.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/vats/src/core/chain-behaviors.js b/packages/vats/src/core/chain-behaviors.js
index f75759fb64f..d5bc221f797 100644
--- a/packages/vats/src/core/chain-behaviors.js
+++ b/packages/vats/src/core/chain-behaviors.js
@@ -312,6 +312,7 @@ export const makeChainStorage = async ({
     bridgeManager,
     BRIDGE_ID.STORAGE,
     ROOT_PATH,
+    { sequence: true },
   );
   chainStorageP.resolve(rootNodeP);
 };

From ade09fbad8c2896614845840ad9c3bfba4eb3243 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 19:46:22 -0400
Subject: [PATCH 08/20] test(scripts): Add test files for
 get-flattened-publication.sh

---
 .../test-get-flattened-publication/bin/curl   | 20 +++++++++++++++++
 .../fixtures/capdata.json                     |  4 ++++
 .../fixtures/flattened-naked.json             |  1 +
 .../fixtures/flattened-streamcell.json        |  2 ++
 .../test-get-flattened-publication/test.sh    | 22 +++++++++++++++++++
 5 files changed, 49 insertions(+)
 create mode 100755 scripts/test/test-get-flattened-publication/bin/curl
 create mode 100644 scripts/test/test-get-flattened-publication/fixtures/capdata.json
 create mode 100644 scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json
 create mode 100644 scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json
 create mode 100755 scripts/test/test-get-flattened-publication/test.sh

diff --git a/scripts/test/test-get-flattened-publication/bin/curl b/scripts/test/test-get-flattened-publication/bin/curl
new file mode 100755
index 00000000000..69bd6e77d5e
--- /dev/null
+++ b/scripts/test/test-get-flattened-publication/bin/curl
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Mock RPC node responses.
+FILE="$(realpath "$BASH_SOURCE")"
+cd "$(dirname "$FILE")/.."
+
+case "${RESPONSE}" in
+  NAKED)
+    # Emit a single naked value.
+    jq . fixtures/capdata.json
+    ;;
+  STREAM_CELL)
+    # Emit a two-result stream cell.
+    jq '{ height: "41", values: [., .] | map(tojson) }' fixtures/capdata.json
+    ;;
+  *)
+    printf 'Missing/invalid RESPONSE environment variable: %s\n' "$RESPONSE" >&2
+    ;;
+esac | \
+# Encode the result to mimic an RPC node response.
+jq '{ value: . | tojson } | tojson | @base64 | { result: { response: { height: "42", value: . } } }'
diff --git a/scripts/test/test-get-flattened-publication/fixtures/capdata.json b/scripts/test/test-get-flattened-publication/fixtures/capdata.json
new file mode 100644
index 00000000000..71270b6fb58
--- /dev/null
+++ b/scripts/test/test-get-flattened-publication/fixtures/capdata.json
@@ -0,0 +1,4 @@
+{
+  "slots": ["slot-ignored", "slot-bar"],
+  "body":"{\"json\":{\"booleans\":[false,true],\"null\":null,\"strings\":[\"\"],\"numbers\":[0]},\"non-json\":{\"bigint\":{\"@qclass\":\"bigint\",\"digits\":\"0\"},\"slot-reference\":{\"@qclass\":\"slot\",\"index\":1,\"iface\":\"Alleged: Bar brand\"}}}"
+}
diff --git a/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json b/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json
new file mode 100644
index 00000000000..891474cd0ce
--- /dev/null
+++ b/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json
@@ -0,0 +1 @@
+{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slot-reference-id":"slot-bar","non-json-slot-reference-allegedName":"Bar","dataBlockHeight":null,"blockHeight":"42"}
diff --git a/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json b/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json
new file mode 100644
index 00000000000..59daf2a9735
--- /dev/null
+++ b/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json
@@ -0,0 +1,2 @@
+{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slot-reference-id":"slot-bar","non-json-slot-reference-allegedName":"Bar","dataBlockHeight":"41","blockHeight":"42"}
+{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slot-reference-id":"slot-bar","non-json-slot-reference-allegedName":"Bar","dataBlockHeight":"41","blockHeight":"42"}
diff --git a/scripts/test/test-get-flattened-publication/test.sh b/scripts/test/test-get-flattened-publication/test.sh
new file mode 100755
index 00000000000..b5128c2322a
--- /dev/null
+++ b/scripts/test/test-get-flattened-publication/test.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+FILE="$(realpath "$BASH_SOURCE")"
+cd "$(dirname "$FILE")"
+export PATH="./bin:$PATH"
+
+failed=0
+out="$(RESPONSE=NAKED ../../get-flattened-publication.sh HOST STORAGE_KEY)"
+if [ "$out" != "$(cat fixtures/flattened-naked.json)" ]; then
+  failed=1
+  echo 'Output did not match expectations for a naked result.'
+  printf '%s\n' "$out" | ${DIFF:-diff -u} ${DIFF_OPTS:-} fixtures/flattened-naked.json - || true
+fi
+out="$(RESPONSE=STREAM_CELL ../../get-flattened-publication.sh HOST STORAGE_KEY)"
+if [ "$out" != "$(cat fixtures/flattened-streamcell.json)" ]; then
+  failed=1
+  echo 'Output did not match expectations for a stream cell result.'
+  printf '%s\n' "$out" | ${DIFF:-diff -u} ${DIFF_OPTS:-} fixtures/flattened-streamcell.json - || true
+fi
+
+[ $failed = 1 ] && exit 1
+echo 'get-flattened-publication.sh tests passed!' >&2
+

From 47fe9fad4e5babe5e6fa61505031971768a1ae74 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 19:47:10 -0400
Subject: [PATCH 09/20] fix(scripts): Fix flattening of falsy values

---
 scripts/get-flattened-publication.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/get-flattened-publication.sh b/scripts/get-flattened-publication.sh
index 6f6060b3168..4e491a247a2 100755
--- a/scripts/get-flattened-publication.sh
+++ b/scripts/get-flattened-publication.sh
@@ -101,7 +101,7 @@ jq -c '
   ) |
 
   # Flatten the resulting structure, joining deep member names with "-".
-  [ paths(scalars) as $path | { key: $path | join("-"), value: getpath($path) } ] | from_entries |
+  [ paths(scalars==.) as $path | { key: $path | join("-"), value: getpath($path) } ] | from_entries |
 
   # Add block height information.
   (.dataBlockHeight |= $dataHeight) |

From da04888e36c33f9c42fb980ab1a7780e8c297387 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 12 Aug 2022 23:10:51 -0400
Subject: [PATCH 10/20] style(casting): Accept review suggestions

---
 packages/casting/src/follower-cosmjs.js  | 29 +++++++++---------------
 packages/casting/test/fake-rpc-server.js |  6 ++---
 2 files changed, 14 insertions(+), 21 deletions(-)

diff --git a/packages/casting/src/follower-cosmjs.js b/packages/casting/src/follower-cosmjs.js
index 66f40b34036..43dee58c2bd 100644
--- a/packages/casting/src/follower-cosmjs.js
+++ b/packages/casting/src/follower-cosmjs.js
@@ -341,27 +341,20 @@ export const makeCosmjsFollower = (
       }
       for (let i = 0; i < streamCell.values.length; i += 1) {
         const data = JSON.parse(streamCell.values[i]);
-        const last = i + 1 === streamCell.values.length;
-        if (!unserializer) {
-          /** @type {T} */
-          const value = data;
+        const isLast = i + 1 === streamCell.values.length;
+        const value = /** @type {T} */ (
+          unserializer
+            ? // eslint-disable-next-line no-await-in-loop,@jessie.js/no-nested-await
+              await E(unserializer).unserialize(data)
+            : data
+        );
+        // QUESTION: How would reach a point where this `isValid()` fails,
+        // and what is the proper handling?
+        if (!unserializer || committer.isValid()) {
           committer.commit({ value });
-          if (!last) {
+          if (!isLast) {
             committer = prepareUpdateInOrder();
           }
-          // eslint-disable-next-line no-continue
-          continue;
-        }
-        // eslint-disable-next-line no-await-in-loop,@jessie.js/no-nested-await
-        const value = await E(unserializer).unserialize(data);
-        if (!committer.isValid()) {
-          // QUESTION: How would we get here, and what is the proper handling?
-          // eslint-disable-next-line no-continue
-          continue;
-        }
-        committer.commit({ value });
-        if (!last) {
-          committer = prepareUpdateInOrder();
         }
       }
     };
diff --git a/packages/casting/test/fake-rpc-server.js b/packages/casting/test/fake-rpc-server.js
index d674bcee572..d7d0c3ee532 100644
--- a/packages/casting/test/fake-rpc-server.js
+++ b/packages/casting/test/fake-rpc-server.js
@@ -64,9 +64,9 @@ const fakeStatusResult = {
   },
 };
 
-/** @typedef {Partial<import('ava').ExecutionContext<{cleanups: Array<() => void>}>> & {context}} fakeServerTestContext */
+/** @typedef {Partial<import('ava').ExecutionContext<{cleanups: Array<() => void>}>> & {context}} FakeServerTestContext */
 /**
- * @param {fakeServerTestContext} t
+ * @param {FakeServerTestContext} t
  * @param {Array<{any}>} fakeValues
  * @param {object} [options]
  * @param {Marshaller} [options.marshaller]
@@ -250,7 +250,7 @@ export const develop = async () => {
       unserialize({ body: jsonMarshalled, slots: [] }),
     ),
   );
-  const mockT = /** @type {fakeServerTestContext} */ (
+  const mockT = /** @type {FakeServerTestContext} */ (
     /** @type {unknown} */ ({
       log: console.log,
       context: { cleanups: [] },

From 0b97c2123098f89b34620a7e278057a9cfee5de3 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Sun, 14 Aug 2022 15:51:12 -0400
Subject: [PATCH 11/20] refactor(scripts): Improve error reporting

---
 scripts/get-flattened-publication.sh | 64 +++++++++++++++++-----------
 1 file changed, 39 insertions(+), 25 deletions(-)

diff --git a/scripts/get-flattened-publication.sh b/scripts/get-flattened-publication.sh
index 4e491a247a2..94e1efea83a 100755
--- a/scripts/get-flattened-publication.sh
+++ b/scripts/get-flattened-publication.sh
@@ -32,7 +32,7 @@ jq --version | awk '
   }
 '
 
-# Make the abci_query request and extract data from a response.
+# Make the abci_query request.
 # cf. https://docs.tendermint.com/master/rpc/#/ABCI/abci_query
 #
 #   {
@@ -46,29 +46,39 @@ jq --version | awk '
 #       }
 #     }
 #   }
+resp="$(
+  # Avoid the GET interface in case interpretation of `path` as JSON is ever fixed.
+  # https://github.com/tendermint/tendermint/issues/9164
+  # curl -sS "${URL_PREFIX%/}/abci_query?path=%22/custom/vstorage/data/$STORAGE_KEY%22" | \
+  curl -sS "$URL_PREFIX" --request POST --header 'Content-Type: application/json' --data "$(
+    printf '{
+      "jsonrpc": "2.0",
+      "id": -1,
+      "method": "abci_query",
+      "params": { "path": "/custom/vstorage/data/%s" }
+    }' "$STORAGE_KEY"
+  )"
+)"
+
+# Decode the response
+response_height_json="$(printf '%s' "$resp" | jq '.result.response.height?' | grep -E '^"[0-9]+"$')"
+if [ ":$response_height_json" = : ]; then
+  printf 'Unable to read response block height:\n%s\n' "$resp" >&2
+  exit 1
+fi
+response_value_json="$(printf '%s' "$resp" | jq '.result.response.value | @base64d | fromjson')"
+if [ ":$response_value_json" = : ]; then
+  printf 'Unable to read response value as base64-encoded JSON text:\n%s\n' "$resp" >&2
+  exit 1
+fi
+unwrapped_value="$(printf '%s' "$response_value_json" | jq '.value | fromjson')"
+if [ ":$unwrapped_value" = : ]; then
+  printf 'Unable to unwrap response:\n%s\n' "$response_value_json" >&2
+  exit 1
+fi
 
-# Avoid the GET interface in case interpretation of `path` as JSON is ever fixed.
-# https://github.com/tendermint/tendermint/issues/9164
-# curl -sS "${URL_PREFIX%/}/abci_query?path=%22/custom/vstorage/data/$STORAGE_KEY%22" | \
-curl -sS "$URL_PREFIX" --request POST --header 'Content-Type: application/json' --data "$(
-  printf '{
-    "jsonrpc": "2.0",
-    "id": -1,
-    "method": "abci_query",
-    "params": { "path": "/custom/vstorage/data/%s" }
-  }' "$STORAGE_KEY"
-)" | \
-# Decode, simplify, flatten, and compact the output.
-jq -c '
-  # Capture response block height.
-  .result.response.height? as $responseHeight |
-
-  # Decode `value` as base64, then decode that as JSON.
-  .result.response.value | @base64d | fromjson |
-
-  # Decode `value` as JSON.
-  .value | fromjson |
-
+# Simplify, flatten, and compact.
+printf '%s' "$unwrapped_value" | jq -c --arg responseHeightJson "$response_height_json" '
   # Upgrade a naked value to a stream cell if necessary.
   if has("height") and has("values") then . else { values: [ . | tojson ] } end |
 
@@ -105,5 +115,9 @@ jq -c '
 
   # Add block height information.
   (.dataBlockHeight |= $dataHeight) |
-  (.blockHeight |= $responseHeight)
-'
+  (.blockHeight |= ($responseHeightJson | fromjson))
+' || {
+  status=$?
+  printf 'Unable to process response value:\n%s\n' "$unwrapped_value" >&2
+  exit $status
+}

From a167271f7920d1dabb8660da597b23289e404e8e Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Sun, 14 Aug 2022 16:31:41 -0400
Subject: [PATCH 12/20] test(scripts): Collapse DIFF_OPTS into DIFF

---
 scripts/test/test-get-flattened-publication/test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/test/test-get-flattened-publication/test.sh b/scripts/test/test-get-flattened-publication/test.sh
index b5128c2322a..70a0fec332e 100755
--- a/scripts/test/test-get-flattened-publication/test.sh
+++ b/scripts/test/test-get-flattened-publication/test.sh
@@ -8,13 +8,13 @@ out="$(RESPONSE=NAKED ../../get-flattened-publication.sh HOST STORAGE_KEY)"
 if [ "$out" != "$(cat fixtures/flattened-naked.json)" ]; then
   failed=1
   echo 'Output did not match expectations for a naked result.'
-  printf '%s\n' "$out" | ${DIFF:-diff -u} ${DIFF_OPTS:-} fixtures/flattened-naked.json - || true
+  printf '%s\n' "$out" | ${DIFF:-diff -u} fixtures/flattened-naked.json - || true
 fi
 out="$(RESPONSE=STREAM_CELL ../../get-flattened-publication.sh HOST STORAGE_KEY)"
 if [ "$out" != "$(cat fixtures/flattened-streamcell.json)" ]; then
   failed=1
   echo 'Output did not match expectations for a stream cell result.'
-  printf '%s\n' "$out" | ${DIFF:-diff -u} ${DIFF_OPTS:-} fixtures/flattened-streamcell.json - || true
+  printf '%s\n' "$out" | ${DIFF:-diff -u} fixtures/flattened-streamcell.json - || true
 fi
 
 [ $failed = 1 ] && exit 1

From 38ce6ea8ee4d66f22f16d04d0823db59ad35173d Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Sun, 14 Aug 2022 16:32:36 -0400
Subject: [PATCH 13/20] test(scripts): Add tests for repeated references of the
 same slot

---
 .../test/test-get-flattened-publication/fixtures/capdata.json | 2 +-
 .../fixtures/flattened-naked.json                             | 2 +-
 .../fixtures/flattened-streamcell.json                        | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/test/test-get-flattened-publication/fixtures/capdata.json b/scripts/test/test-get-flattened-publication/fixtures/capdata.json
index 71270b6fb58..b3d0d65593b 100644
--- a/scripts/test/test-get-flattened-publication/fixtures/capdata.json
+++ b/scripts/test/test-get-flattened-publication/fixtures/capdata.json
@@ -1,4 +1,4 @@
 {
   "slots": ["slot-ignored", "slot-bar"],
-  "body":"{\"json\":{\"booleans\":[false,true],\"null\":null,\"strings\":[\"\"],\"numbers\":[0]},\"non-json\":{\"bigint\":{\"@qclass\":\"bigint\",\"digits\":\"0\"},\"slot-reference\":{\"@qclass\":\"slot\",\"index\":1,\"iface\":\"Alleged: Bar brand\"}}}"
+  "body":"{\"json\":{\"booleans\":[false,true],\"null\":null,\"strings\":[\"\"],\"numbers\":[0]},\"non-json\":{\"bigint\":{\"@qclass\":\"bigint\",\"digits\":\"0\"},\"slotref\":{\"@qclass\":\"slot\",\"index\":1,\"iface\":\"Alleged: Bar brand\"},\"slotreref\":{\"@qclass\":\"slot\",\"index\":1}}}"
 }
diff --git a/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json b/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json
index 891474cd0ce..3e85dc109f9 100644
--- a/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json
+++ b/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json
@@ -1 +1 @@
-{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slot-reference-id":"slot-bar","non-json-slot-reference-allegedName":"Bar","dataBlockHeight":null,"blockHeight":"42"}
+{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slotref-id":"slot-bar","non-json-slotref-allegedName":"Bar","non-json-slotreref-id":"slot-bar","non-json-slotreref-allegedName":"Bar","dataBlockHeight":null,"blockHeight":"42"}
diff --git a/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json b/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json
index 59daf2a9735..e6fa82f160f 100644
--- a/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json
+++ b/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json
@@ -1,2 +1,2 @@
-{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slot-reference-id":"slot-bar","non-json-slot-reference-allegedName":"Bar","dataBlockHeight":"41","blockHeight":"42"}
-{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slot-reference-id":"slot-bar","non-json-slot-reference-allegedName":"Bar","dataBlockHeight":"41","blockHeight":"42"}
+{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slotref-id":"slot-bar","non-json-slotref-allegedName":"Bar","non-json-slotreref-id":"slot-bar","non-json-slotreref-allegedName":"Bar","dataBlockHeight":"41","blockHeight":"42"}
+{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slotref-id":"slot-bar","non-json-slotref-allegedName":"Bar","non-json-slotreref-id":"slot-bar","non-json-slotreref-allegedName":"Bar","dataBlockHeight":"41","blockHeight":"42"}

From bf1f29a81ecc37e9b06447bb0632f454bdad9cfe Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Sun, 14 Aug 2022 16:33:20 -0400
Subject: [PATCH 14/20] fix(scripts): Default slot iface from prior use

---
 scripts/get-flattened-publication.sh | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/scripts/get-flattened-publication.sh b/scripts/get-flattened-publication.sh
index 94e1efea83a..12586d2caec 100755
--- a/scripts/get-flattened-publication.sh
+++ b/scripts/get-flattened-publication.sh
@@ -88,8 +88,19 @@ printf '%s' "$unwrapped_value" | jq -c --arg responseHeightJson "$response_heigh
   # Flatten each value independently.
   .values[] | fromjson |
 
-  # Capture `slots`, then decode `body` as JSON.
-  .slots as $slots | .body | fromjson |
+  # Capture `slots` values.
+  .slots as $slotValues |
+
+  # Decode `body` as JSON.
+  .body | fromjson |
+
+  # Capture slot names (which generally appear only at first reference).
+  (
+    [ .. | select(type=="object" and .["@qclass"]=="slot" and (.iface | type=="string")) ] |
+    map({ key: .index | tostring, value: .iface }) |
+    unique_by(.key) |
+    from_entries
+  ) as $slotNames |
 
   # Replace select capdata.
   walk(
@@ -99,11 +110,11 @@ printf '%s' "$unwrapped_value" | jq -c --arg responseHeightJson "$response_heigh
     elif type=="object" and .["@qclass"]=="slot" then
       # Replace slot reference capdata with {
       #   id: <value from global `slots`>,
-      #   allegedName: <extracted from local `iface`>,
+      #   allegedName: <extracted from local or previous `iface`>,
       # }.
       {
-        id: $slots[.index],
-        allegedName: .iface | sub("^Alleged: (?<name>.*) brand$"; "\(.name)"; "m")
+        id: $slotValues[.index],
+        allegedName: (try ((.iface // $slotNames[.index | tostring]) | sub("^Alleged: (?<name>.*) brand$"; "\(.name)"; "m")) catch null)
       }
     else
       .

From c786022394a6d4842b2a6b0397e05daa821d65cd Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Sun, 14 Aug 2022 17:10:27 -0400
Subject: [PATCH 15/20] test(scripts): Add a scripts test runner

---
 .github/workflows/test-scripts.yml | 14 ++++++++++++++
 scripts/test/README.md             |  6 ++++++
 scripts/test/test.sh               | 11 +++++++++++
 3 files changed, 31 insertions(+)
 create mode 100644 .github/workflows/test-scripts.yml
 create mode 100644 scripts/test/README.md
 create mode 100755 scripts/test/test.sh

diff --git a/.github/workflows/test-scripts.yml b/.github/workflows/test-scripts.yml
new file mode 100644
index 00000000000..b3dadbc6d7a
--- /dev/null
+++ b/.github/workflows/test-scripts.yml
@@ -0,0 +1,14 @@
+name: Run scripts tests
+
+on:
+  pull_request:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-scripts-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: scripts/test/test.sh
diff --git a/scripts/test/README.md b/scripts/test/README.md
new file mode 100644
index 00000000000..c28f22302e0
--- /dev/null
+++ b/scripts/test/README.md
@@ -0,0 +1,6 @@
+This directory contains shell-based tests for scripts.
+
+test.sh looks for a test.sh in each subdirectory and executes it,
+expecting an exit status code of 0 to indicate success.
+
+It exit with status 0 if all tests pass and status 1 if any fail.
diff --git a/scripts/test/test.sh b/scripts/test/test.sh
new file mode 100755
index 00000000000..8ba4df6b0f2
--- /dev/null
+++ b/scripts/test/test.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+FILE="$(realpath "$BASH_SOURCE")"
+cd "$(dirname "$FILE")"
+
+# Execute each test.sh in a subdirectory
+# and exit successfully if and only if they all succeed.
+fail=0
+for test in $(find * -mindepth 1 -maxdepth 1 -name test.sh); do
+	"$test" && echo "OK $test" || { echo "FAIL $test"; fail=1; }
+done
+exit $fail

From 929a07c581d9ec7af5a21d7d321a027f5e07ab83 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Wed, 17 Aug 2022 11:59:33 -0400
Subject: [PATCH 16/20] chore: Commit to stream cell values being strings

---
 golang/cosmos/x/vstorage/keeper/keeper.go | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/golang/cosmos/x/vstorage/keeper/keeper.go b/golang/cosmos/x/vstorage/keeper/keeper.go
index b9e7e3c546b..b17a4e33d56 100644
--- a/golang/cosmos/x/vstorage/keeper/keeper.go
+++ b/golang/cosmos/x/vstorage/keeper/keeper.go
@@ -16,13 +16,10 @@ import (
 // StreamCell is an envelope representing a sequence of values written at a path in a single block.
 // It is persisted to storage as a { "height": "<digits>", "values": ["...", ...] } JSON text
 // that off-chain consumers rely upon.
+// Many of those consumers *also* rely upon the strings of "values" being valid JSON text
+// (cf. scripts/get-flattened-publication.sh), but we do not enforce that in this package.
 type StreamCell struct {
-	Height string `json:"height"`
-	// XXX Should Values be []string or []interface{}?
-	// The latter would remove a layer of JSON encoding (e.g., `[{…}]` rather than `["{…}"]`,
-	// but would add a requirement exclusive to AppendStorageValueAndNotify that its input be JSON.
-	// On the other hand, we could always extend this format in the future to include an indication
-	// that values are subject to a different encoding, e.g. `"valueEncoding":"base64"`.
+	Height string   `json:"height"`
 	Values []string `json:"values"`
 }
 

From 40530fd7a1b925fc13d8bb02c4e28ca87d9d1c8e Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Thu, 18 Aug 2022 14:44:34 -0400
Subject: [PATCH 17/20] refactor: Rename stream cell "height" to "blockHeight"

---
 golang/cosmos/x/vstorage/keeper/keeper.go           | 13 ++++++-------
 packages/casting/src/follower-cosmjs.js             |  2 +-
 packages/casting/test/fake-rpc-server.js            |  8 ++++----
 scripts/get-flattened-publication.sh                |  4 ++--
 .../test/test-get-flattened-publication/bin/curl    |  2 +-
 5 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/golang/cosmos/x/vstorage/keeper/keeper.go b/golang/cosmos/x/vstorage/keeper/keeper.go
index b17a4e33d56..f393fc63104 100644
--- a/golang/cosmos/x/vstorage/keeper/keeper.go
+++ b/golang/cosmos/x/vstorage/keeper/keeper.go
@@ -14,13 +14,13 @@ import (
 )
 
 // StreamCell is an envelope representing a sequence of values written at a path in a single block.
-// It is persisted to storage as a { "height": "<digits>", "values": ["...", ...] } JSON text
+// It is persisted to storage as a { "blockHeight": "<digits>", "values": ["...", ...] } JSON text
 // that off-chain consumers rely upon.
 // Many of those consumers *also* rely upon the strings of "values" being valid JSON text
 // (cf. scripts/get-flattened-publication.sh), but we do not enforce that in this package.
 type StreamCell struct {
-	Height string   `json:"height"`
-	Values []string `json:"values"`
+	BlockHeight string   `json:"blockHeight"`
+	Values      []string `json:"values"`
 }
 
 // Keeper maintains the link to data storage and exposes getter/setter methods
@@ -140,16 +140,15 @@ func (k Keeper) SetStorageAndNotify(ctx sdk.Context, path, value string) {
 }
 
 func (k Keeper) AppendStorageValueAndNotify(ctx sdk.Context, path, value string) error {
-	height := strconv.FormatInt(ctx.BlockHeight(), 10)
+	blockHeight := strconv.FormatInt(ctx.BlockHeight(), 10)
 
 	// Preserve correctly-formatted data within the current block,
 	// otherwise initialize a blank cell.
 	currentData := k.GetData(ctx, path)
 	var cell StreamCell
 	_ = json.Unmarshal([]byte(currentData), &cell)
-	if cell.Height != height {
-		cell.Height = height
-		cell.Values = make([]string, 0, 1)
+	if cell.BlockHeight != blockHeight {
+		cell = StreamCell{BlockHeight: blockHeight, Values: make([]string, 0, 1)}
 	}
 
 	// Append the new value.
diff --git a/packages/casting/src/follower-cosmjs.js b/packages/casting/src/follower-cosmjs.js
index 43dee58c2bd..58dee7540e7 100644
--- a/packages/casting/src/follower-cosmjs.js
+++ b/packages/casting/src/follower-cosmjs.js
@@ -336,7 +336,7 @@ export const makeCosmjsFollower = (
       lastBuf = buf;
       let streamCell = decode(buf);
       // Upgrade a naked value to a JSON stream cell if necessary.
-      if (!streamCell.height || !streamCell.values) {
+      if (!streamCell.blockHeight || !streamCell.values) {
         streamCell = { values: [JSON.stringify(streamCell)] };
       }
       for (let i = 0; i < streamCell.values.length; i += 1) {
diff --git a/packages/casting/test/fake-rpc-server.js b/packages/casting/test/fake-rpc-server.js
index d7d0c3ee532..4c1469204bf 100644
--- a/packages/casting/test/fake-rpc-server.js
+++ b/packages/casting/test/fake-rpc-server.js
@@ -106,7 +106,7 @@ export const startFakeServer = (t, fakeValues, options = {}) => {
       buf.set(ascii, dataPrefix.length);
       return toBase64(buf);
     };
-    let height = 74863;
+    let blockHeight = 74863;
     let responseValueBase64;
     app.post('/tendermint-rpc', (req, res) => {
       log('received', req.path, req.body, req.params);
@@ -125,7 +125,7 @@ export const startFakeServer = (t, fakeValues, options = {}) => {
           break;
         }
         case 'abci_query': {
-          height += 2;
+          blockHeight += 2;
           const values = fakeValues.splice(0, Math.max(1, batchSize));
           if (values.length > 0) {
             if (batchSize > 0) {
@@ -134,7 +134,7 @@ export const startFakeServer = (t, fakeValues, options = {}) => {
                 JSON.stringify(marshaller.serialize(val)),
               );
               responseValueBase64 = encode({
-                height: String(height - 1),
+                blockHeight: String(blockHeight - 1),
                 values: serializedValues,
               });
             } else {
@@ -153,7 +153,7 @@ export const startFakeServer = (t, fakeValues, options = {}) => {
               ).toString('base64'),
               value: responseValueBase64,
               proofOps: null,
-              height: String(height),
+              height: String(blockHeight),
               codespace: '',
             },
           };
diff --git a/scripts/get-flattened-publication.sh b/scripts/get-flattened-publication.sh
index 12586d2caec..4e872c462e7 100755
--- a/scripts/get-flattened-publication.sh
+++ b/scripts/get-flattened-publication.sh
@@ -80,10 +80,10 @@ fi
 # Simplify, flatten, and compact.
 printf '%s' "$unwrapped_value" | jq -c --arg responseHeightJson "$response_height_json" '
   # Upgrade a naked value to a stream cell if necessary.
-  if has("height") and has("values") then . else { values: [ . | tojson ] } end |
+  if has("blockHeight") and has("values") then . else { values: [ . | tojson ] } end |
 
   # Capture data block height.
-  .height as $dataHeight |
+  .blockHeight as $dataHeight |
 
   # Flatten each value independently.
   .values[] | fromjson |
diff --git a/scripts/test/test-get-flattened-publication/bin/curl b/scripts/test/test-get-flattened-publication/bin/curl
index 69bd6e77d5e..b56e4111566 100755
--- a/scripts/test/test-get-flattened-publication/bin/curl
+++ b/scripts/test/test-get-flattened-publication/bin/curl
@@ -10,7 +10,7 @@ case "${RESPONSE}" in
     ;;
   STREAM_CELL)
     # Emit a two-result stream cell.
-    jq '{ height: "41", values: [., .] | map(tojson) }' fixtures/capdata.json
+    jq '{ blockHeight: "41", values: [., .] | map(tojson) }' fixtures/capdata.json
     ;;
   *)
     printf 'Missing/invalid RESPONSE environment variable: %s\n' "$RESPONSE" >&2

From e4da51512d1583d14ab44eda880e0764b0f1d167 Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 19 Aug 2022 12:52:40 -0400
Subject: [PATCH 18/20] refactor(casting): Synchronize stream cell detection
 with scripts/get-flattened-publication.sh

Use existence of "blockHeight" and "values" properties rather than truthiness
(which differs between JavaScript and `jq`).
---
 packages/casting/src/follower-cosmjs.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/casting/src/follower-cosmjs.js b/packages/casting/src/follower-cosmjs.js
index 58dee7540e7..5de9286c457 100644
--- a/packages/casting/src/follower-cosmjs.js
+++ b/packages/casting/src/follower-cosmjs.js
@@ -336,7 +336,10 @@ export const makeCosmjsFollower = (
       lastBuf = buf;
       let streamCell = decode(buf);
       // Upgrade a naked value to a JSON stream cell if necessary.
-      if (!streamCell.blockHeight || !streamCell.values) {
+      if (
+        streamCell.blockHeight === undefined ||
+        streamCell.values === undefined
+      ) {
         streamCell = { values: [JSON.stringify(streamCell)] };
       }
       for (let i = 0; i < streamCell.values.length; i += 1) {

From 5328671f936abbc9fba98205af9a1820a0ff29fc Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 19 Aug 2022 14:53:54 -0400
Subject: [PATCH 19/20] chore: Align with latest Tendermint RPC documentation

---
 scripts/get-flattened-publication.sh | 31 ++++++++++++++--------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/scripts/get-flattened-publication.sh b/scripts/get-flattened-publication.sh
index 4e872c462e7..543b3cdeecb 100755
--- a/scripts/get-flattened-publication.sh
+++ b/scripts/get-flattened-publication.sh
@@ -33,19 +33,7 @@ jq --version | awk '
 '
 
 # Make the abci_query request.
-# cf. https://docs.tendermint.com/master/rpc/#/ABCI/abci_query
-#
-#   {
-#     "jsonrpc": "2.0",
-#     "id": <integer>,
-#     "result": {
-#       "response": {
-#         "value": "<base64-encoded JSON text>",
-#         "height": "<decimal digits>",
-#         ...
-#       }
-#     }
-#   }
+# cf. https://docs.tendermint.com/v0.34/rpc/
 resp="$(
   # Avoid the GET interface in case interpretation of `path` as JSON is ever fixed.
   # https://github.com/tendermint/tendermint/issues/9164
@@ -53,14 +41,27 @@ resp="$(
   curl -sS "$URL_PREFIX" --request POST --header 'Content-Type: application/json' --data "$(
     printf '{
       "jsonrpc": "2.0",
-      "id": -1,
+      "id": 1,
       "method": "abci_query",
       "params": { "path": "/custom/vstorage/data/%s" }
     }' "$STORAGE_KEY"
   )"
 )"
 
-# Decode the response
+# Decode the response.
+# cf. https://docs.tendermint.com/master/rpc/#/ABCI/abci_query
+#
+#   {
+#     "jsonrpc": "2.0",
+#     "id": <integer>,
+#     "result": {
+#       "response": {
+#         "height": "<decimal digits>",
+#         "value": "<base64-encoded JSON text>",
+#         ...
+#       }
+#     }
+#   }
 response_height_json="$(printf '%s' "$resp" | jq '.result.response.height?' | grep -E '^"[0-9]+"$')"
 if [ ":$response_height_json" = : ]; then
   printf 'Unable to read response block height:\n%s\n' "$resp" >&2

From 2ce41da587de719660a172ff99abc7e1c24bb81f Mon Sep 17 00:00:00 2001
From: Richard Gibson <richard.gibson@gmail.com>
Date: Fri, 19 Aug 2022 18:19:09 -0400
Subject: [PATCH 20/20] refactor: Rename get-flattened-publication.sh
 "blockHeight" to "responseBlockHeight"

---
 scripts/get-flattened-publication.sh                          | 2 +-
 .../fixtures/flattened-naked.json                             | 2 +-
 .../fixtures/flattened-streamcell.json                        | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/get-flattened-publication.sh b/scripts/get-flattened-publication.sh
index 543b3cdeecb..2c93016eccd 100755
--- a/scripts/get-flattened-publication.sh
+++ b/scripts/get-flattened-publication.sh
@@ -127,7 +127,7 @@ printf '%s' "$unwrapped_value" | jq -c --arg responseHeightJson "$response_heigh
 
   # Add block height information.
   (.dataBlockHeight |= $dataHeight) |
-  (.blockHeight |= ($responseHeightJson | fromjson))
+  (.responseBlockHeight |= ($responseHeightJson | fromjson))
 ' || {
   status=$?
   printf 'Unable to process response value:\n%s\n' "$unwrapped_value" >&2
diff --git a/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json b/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json
index 3e85dc109f9..74dc9294d0a 100644
--- a/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json
+++ b/scripts/test/test-get-flattened-publication/fixtures/flattened-naked.json
@@ -1 +1 @@
-{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slotref-id":"slot-bar","non-json-slotref-allegedName":"Bar","non-json-slotreref-id":"slot-bar","non-json-slotreref-allegedName":"Bar","dataBlockHeight":null,"blockHeight":"42"}
+{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slotref-id":"slot-bar","non-json-slotref-allegedName":"Bar","non-json-slotreref-id":"slot-bar","non-json-slotreref-allegedName":"Bar","dataBlockHeight":null,"responseBlockHeight":"42"}
diff --git a/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json b/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json
index e6fa82f160f..1450617ecc1 100644
--- a/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json
+++ b/scripts/test/test-get-flattened-publication/fixtures/flattened-streamcell.json
@@ -1,2 +1,2 @@
-{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slotref-id":"slot-bar","non-json-slotref-allegedName":"Bar","non-json-slotreref-id":"slot-bar","non-json-slotreref-allegedName":"Bar","dataBlockHeight":"41","blockHeight":"42"}
-{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slotref-id":"slot-bar","non-json-slotref-allegedName":"Bar","non-json-slotreref-id":"slot-bar","non-json-slotreref-allegedName":"Bar","dataBlockHeight":"41","blockHeight":"42"}
+{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slotref-id":"slot-bar","non-json-slotref-allegedName":"Bar","non-json-slotreref-id":"slot-bar","non-json-slotreref-allegedName":"Bar","dataBlockHeight":"41","responseBlockHeight":"42"}
+{"json-booleans-0":false,"json-booleans-1":true,"json-null":null,"json-strings-0":"","json-numbers-0":0,"non-json-bigint":"0","non-json-slotref-id":"slot-bar","non-json-slotref-allegedName":"Bar","non-json-slotreref-id":"slot-bar","non-json-slotreref-allegedName":"Bar","dataBlockHeight":"41","responseBlockHeight":"42"}