Feature/optimize collections for json (#38)

* Optimize dict for JSON

* Refactor grid

* Move column caching into HGrid implementation

* Refactor grid

* Add grid stores

* Add refreshColumns to grid

* Manually refresh columns in a grid

* Remove dict validate

* Tweak grid toZinc

* Remove observable hack

* Add tests for grid json encoding

* Add lazy grid json store

* Tweak index

* Add toJSONString to all hval types

* Ensure dict lazily decodes gradually

* Add JSON string store implementations for grid and dict

* Encode to JSON byte buffer

* Add JSON byte buffer support to grid and dict store

* Add Uint8Array buffer stores

* Move list

* Make symbols readonly

* Add list store

* Add list JSON handling

* Add list JSON store

* Optimize JSON encoding of a grid

* Tweak JSON list encoding

* Ensure dict json store saves on memory

* Tweak position of dict store symbol

* Refactor grid data handling

* Tweak list json save memory

* Add list JSON string store

* Add list byte buffer support

* Add performance test

* Remove unnecessary toJSON

* Fix test comments

package.json

@@ -33,9 +33,9 @@
 		"bundle": "webpack",
 		"start": "npm install && npm run test && npm run build && npm run bundle && npm run server",
 		"circular": "madge --circular --extensions ts ./",
-		"examples": "nodemon --ext ts --ignore dist/ --exec \"ts-node ./examples/defs.ts\"",
 		"prepack": "npm run lint && npm test && npm run build",
-		"checktypes": "tsc --noEmit"
+		"checktypes": "tsc --noEmit",
+		"perf:readgrid": "node ./perf/readGrid.js"
 	},
 	"files": [
 		"dist/**/*"

perf/readGrid.js

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2025, J2 Innovations. All Rights Reserved
+ */
+
+const { readFile } = require('fs/promises')
+const { HGrid, GridJsonStore } = require('../dist/index.es')
+
+async function run() {
+	const file = await readFile('./spec/files/skySparkDefs.json', 'utf-8')
+	const json = JSON.parse(file)
+
+	function doGrid(grid) {
+		grid.meta.set('foo', 'bar')
+
+		grid.toJSON()
+	}
+
+	function testOld() {
+		const grid = HGrid.make({
+			meta: json.meta,
+			rows: json.rows,
+			columns: json.cols,
+		})
+		doGrid(grid)
+	}
+
+	function testNew() {
+		const grid = new HGrid(new GridJsonStore(json))
+		doGrid(grid)
+	}
+
+	console.log('Priming (triggers JIT)...')
+	for (let i = 0; i < 10_000; ++i) {
+		testOld()
+		testNew()
+	}
+
+	console.log('Running tests...')
+	{
+		const t0 = performance.now()
+		testOld()
+		const t1 = performance.now()
+		console.log('Performance test old way: ' + (t1 - t0) + ' milliseconds.')
+	}
+
+	{
+		let t0 = performance.now()
+		testNew()
+		const t1 = performance.now()
+		console.log('Performance test new way: ' + (t1 - t0) + ' milliseconds.')
+	}
+}
+
+run()

spec/core/Array.spec.ts

@@ -3,9 +3,9 @@
  */
 
 import '../../src/core/Array'
-import { HList } from '../../src/core/HList'
-import { HDict } from '../../src/core/HDict'
-import { HGrid } from '../../src/core/HGrid'
+import { HList } from '../../src/core/list/HList'
+import { HDict } from '../../src/core/dict/HDict'
+import { HGrid } from '../../src/core/grid/HGrid'
 import { HStr } from '../../src/core/HStr'
 import { HBool } from '../../src/core/HBool'
 import { HNum } from '../../src/core/HNum'

spec/core/EnumTag.spec.ts

@@ -3,8 +3,8 @@
  */
 
 import { HStr } from '../../src/core/HStr'
-import { HGrid } from '../../src/core/HGrid'
-import { HDict } from '../../src/core/HDict'
+import { HGrid } from '../../src/core/grid/HGrid'
+import { HDict } from '../../src/core/dict/HDict'
 import { HNum } from '../../src/core/HNum'
 import { HBool } from '../../src/core/HBool'
 import { EnumTag } from '../../src/core/EnumTag'

spec/core/HBool.spec.ts

@@ -3,10 +3,12 @@
  */
 
 import { HBool } from '../../src/core/HBool'
-import { HGrid } from '../../src/core/HGrid'
-import { HList } from '../../src/core/HList'
-import { HDict } from '../../src/core/HDict'
+import { HGrid } from '../../src/core/grid/HGrid'
+import { HList } from '../../src/core/list/HList'
+import { HDict } from '../../src/core/dict/HDict'
 import { Kind } from '../../src/core/Kind'
+import { TEXT_ENCODER } from '../../src/core/HVal'
+
 import '../matchers'
 import '../customMatchers'
 
@@ -126,6 +128,20 @@ describe('HBool', function (): void {
 		})
 	}) // #toJSON()
 
+	describe('#toJSONString()', function (): void {
+		it('returns a JSON string', function (): void {
+			expect(HBool.make(true).toJSONString()).toEqual('true')
+		})
+	}) // #toJSONString()
+
+	describe('#toJSONUint8Array()', function (): void {
+		it('returns a JSON byte buffer', function (): void {
+			expect(HBool.make(true).toJSONUint8Array()).toEqual(
+				TEXT_ENCODER.encode('true')
+			)
+		})
+	}) // #toJSONUint8Array()
+
 	describe('#toJSONv3()', function (): void {
 		it('returns JSON for false', function (): void {
 			expect(HBool.make(false).toJSONv3()).toBe(false)

spec/core/HCoord.spec.ts

@@ -4,9 +4,11 @@
 
 import { HCoord } from '../../src/core/HCoord'
 import { Kind } from '../../src/core/Kind'
-import { HGrid } from '../../src/core/HGrid'
-import { HList } from '../../src/core/HList'
-import { HDict } from '../../src/core/HDict'
+import { HGrid } from '../../src/core/grid/HGrid'
+import { HList } from '../../src/core/list/HList'
+import { HDict } from '../../src/core/dict/HDict'
+import { TEXT_ENCODER } from '../../src/core/HVal'
+
 import '../matchers'
 import '../customMatchers'
 
@@ -171,6 +173,32 @@ describe('HCoord', function (): void {
 		})
 	}) // #toJSON()
 
+	describe('#toJSONString()', function (): void {
+		it('returns a JSON string', function (): void {
+			expect(makeCoord(2, 4).toJSONString()).toBe(
+				JSON.stringify({
+					_kind: Kind.Coord,
+					lat: 2,
+					lng: 4,
+				})
+			)
+		})
+	}) // #toJSONString()
+
+	describe('#toJSONUint8Array()', function (): void {
+		it('returns a JSON byte buffer', function (): void {
+			expect(makeCoord(2, 4).toJSONUint8Array()).toEqual(
+				TEXT_ENCODER.encode(
+					JSON.stringify({
+						_kind: Kind.Coord,
+						lat: 2,
+						lng: 4,
+					})
+				)
+			)
+		})
+	}) // #toJSONUint8Array()
+
 	describe('#toJSONv3()', function (): void {
 		it('returns a JSON string', function (): void {
 			expect(makeCoord(2, 4).toJSONv3()).toBe('c:2,4')

spec/core/HDate.spec.ts

@@ -4,9 +4,11 @@
 
 import { HDate } from '../../src/core/HDate'
 import { Kind } from '../../src/core/Kind'
-import { HGrid } from '../../src/core/HGrid'
-import { HList } from '../../src/core/HList'
-import { HDict } from '../../src/core/HDict'
+import { HGrid } from '../../src/core/grid/HGrid'
+import { HList } from '../../src/core/list/HList'
+import { HDict } from '../../src/core/dict/HDict'
+import { TEXT_ENCODER } from '../../src/core/HVal'
+
 import '../matchers'
 import '../customMatchers'
 
@@ -170,6 +172,30 @@ describe('HDate', function (): void {
 		})
 	}) // #toJSON()
 
+	describe('#toJSONString()', function (): void {
+		it('returns a JSON string', function (): void {
+			expect(HDate.make('2020-01-01').toJSONString()).toBe(
+				JSON.stringify({
+					_kind: Kind.Date,
+					val: '2020-01-01',
+				})
+			)
+		})
+	}) // #toJSONString()
+
+	describe('#toJSONUint8Array()', function (): void {
+		it('returns a JSON byte buffer', function (): void {
+			expect(HDate.make('2020-01-01').toJSONUint8Array()).toEqual(
+				TEXT_ENCODER.encode(
+					JSON.stringify({
+						_kind: Kind.Date,
+						val: '2020-01-01',
+					})
+				)
+			)
+		})
+	}) // #toJSONUint8Array()
+
 	describe('#toJSONv3()', function (): void {
 		it('returns a string', function (): void {
 			expect(HDate.make('2020-01-01').toJSONv3()).toBe('d:2020-01-01')

spec/core/HDateTime.spec.ts

@@ -4,9 +4,11 @@
 
 import { HDateTime } from '../../src/core/HDateTime'
 import { Kind } from '../../src/core/Kind'
-import { HGrid } from '../../src/core/HGrid'
-import { HList } from '../../src/core/HList'
-import { HDict } from '../../src/core/HDict'
+import { HGrid } from '../../src/core/grid/HGrid'
+import { HList } from '../../src/core/list/HList'
+import { HDict } from '../../src/core/dict/HDict'
+import { TEXT_ENCODER } from '../../src/core/HVal'
+
 import '../matchers'
 import '../customMatchers'
 import { DateTime } from 'luxon'
@@ -205,6 +207,32 @@ describe('HDateTime', function (): void {
 		})
 	}) // #toJSON()
 
+	describe('#toJSONString()', function (): void {
+		it('returns a JSON string', function (): void {
+			expect(HDateTime.make('2009-11-09T15:39:00Z').toJSONString()).toBe(
+				JSON.stringify({
+					_kind: Kind.DateTime,
+					val: '2009-11-09T15:39:00Z',
+				})
+			)
+		})
+	}) // #toJSONString()
+
+	describe('#toJSONUint8Array()', function (): void {
+		it('returns a JSON byte buffer', function (): void {
+			expect(
+				HDateTime.make('2009-11-09T15:39:00Z').toJSONUint8Array()
+			).toEqual(
+				TEXT_ENCODER.encode(
+					JSON.stringify({
+						_kind: Kind.DateTime,
+						val: '2009-11-09T15:39:00Z',
+					})
+				)
+			)
+		})
+	}) // #toJSONUint8Array()
+
 	describe('#toJSONv3()', () => {
 		it('returns a string', function (): void {
 			expect(HDateTime.make('2009-11-09T15:39:00Z').toJSONv3()).toBe(

spec/core/HMarker.spec.ts

@@ -4,9 +4,11 @@
 
 import { HMarker } from '../../src/core/HMarker'
 import { Kind } from '../../src/core/Kind'
-import { HGrid } from '../../src/core/HGrid'
-import { HList } from '../../src/core/HList'
-import { HDict } from '../../src/core/HDict'
+import { HGrid } from '../../src/core/grid/HGrid'
+import { HList } from '../../src/core/list/HList'
+import { HDict } from '../../src/core/dict/HDict'
+import { TEXT_ENCODER } from '../../src/core/HVal'
+
 import '../matchers'
 import '../customMatchers'
 
@@ -75,6 +77,28 @@ describe('HMarker', function (): void {
 		})
 	}) // #toJSON()
 
+	describe('#toJSONString()', function (): void {
+		it('returns a JSON string', function (): void {
+			expect(HMarker.make().toJSONString()).toBe(
+				JSON.stringify({
+					_kind: Kind.Marker,
+				})
+			)
+		})
+	}) // #toJSONString()
+
+	describe('#toJSONUint8Array()', function (): void {
+		it('returns a JSON byte buffer', function (): void {
+			expect(HMarker.make().toJSONUint8Array()).toEqual(
+				TEXT_ENCODER.encode(
+					JSON.stringify({
+						_kind: Kind.Marker,
+					})
+				)
+			)
+		})
+	}) // #toJSONUint8Array()
+
 	describe('#toJSONv3()', function (): void {
 		it('returns JSON', function (): void {
 			expect(HMarker.make().toJSONv3()).toEqual('m:')

spec/core/HNa.spec.ts

@@ -4,9 +4,11 @@
 
 import { HNa } from '../../src/core/HNa'
 import { Kind } from '../../src/core/Kind'
-import { HGrid } from '../../src/core/HGrid'
-import { HList } from '../../src/core/HList'
-import { HDict } from '../../src/core/HDict'
+import { HGrid } from '../../src/core/grid/HGrid'
+import { HList } from '../../src/core/list/HList'
+import { HDict } from '../../src/core/dict/HDict'
+import { TEXT_ENCODER } from '../../src/core/HVal'
+
 import '../matchers'
 import '../customMatchers'
 
@@ -71,6 +73,28 @@ describe('HNa', function (): void {
 		})
 	}) // #toJSON()
 
+	describe('#toJSONString()', function (): void {
+		it('returns a JSON string', function (): void {
+			expect(HNa.make().toJSONString()).toBe(
+				JSON.stringify({
+					_kind: Kind.NA,
+				})
+			)
+		})
+	}) // #toJSONString()
+
+	describe('#toJSONUint8Array()', function (): void {
+		it('returns a JSON byte buffer', function (): void {
+			expect(HNa.make().toJSONUint8Array()).toEqual(
+				TEXT_ENCODER.encode(
+					JSON.stringify({
+						_kind: Kind.NA,
+					})
+				)
+			)
+		})
+	}) // #toJSONUint8Array()
+
 	describe('#toJSONv3()', function (): void {
 		it('returns JSON', function (): void {
 			expect(HNa.make().toJSONv3()).toEqual('z:')

spec/core/HNamespace.spec.ts

@@ -3,13 +3,13 @@
  */
 
 import { HNamespace, Reflection } from '../../src/core/HNamespace'
-import { HGrid } from '../../src/core/HGrid'
-import { HDict } from '../../src/core/HDict'
+import { HGrid } from '../../src/core/grid/HGrid'
+import { HDict } from '../../src/core/dict/HDict'
 import { HSymbol } from '../../src/core/HSymbol'
 import { ZincReader } from '../../src/core/ZincReader'
 import { HRef } from '../../src/core/HRef'
 import { HStr } from '../../src/core/HStr'
-import { HList } from '../../src/core/HList'
+import { HList } from '../../src/core/list/HList'
 import { HMarker } from '../../src/core/HMarker'
 import { HNum } from '../../src/core/HNum'
 import { Kind } from '../../src/core/Kind'