From 87449fcc16f9a439c5aa31ab66fbd82e29794eb9 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Tue, 15 Oct 2024 14:57:29 -0400 Subject: [PATCH] feat(arrow): Experimental module for Apache Arrow attribute data extraction (#2278) --- modules/arrow/README.md | 5 + modules/arrow/package.json | 49 +++++ .../arrow/src/arrow/analyze-arrow-table.ts | 22 +++ modules/arrow/src/arrow/arrow-column-info.ts | 88 +++++++++ modules/arrow/src/arrow/arrow-paths.ts | 115 +++++++++++ modules/arrow/src/arrow/arrow-types.ts | 73 +++++++ modules/arrow/src/index.ts | 16 ++ .../test/arrow/analyze-arrow-table.spec.ts | 21 ++ modules/arrow/test/arrow/arrow-paths.spec.ts | 33 ++++ .../arrow/test/arrow/get-arrow-data.spec.ts | 29 +++ .../test/data/arrow/make-arrow-tables.ts | 64 ++++++ modules/arrow/test/index.ts | 7 + modules/arrow/tsconfig.json | 14 ++ modules/core/src/index.ts | 17 +- .../shadertypes/utils/decode-data-types.ts | 18 ++ .../shadertypes/utils/decode-shader-types.ts | 7 + .../shadertypes/utils/decode-vertex-format.ts | 89 +++++---- modules/core/src/types.ts | 17 +- modules/engine/README.md | 5 + test/modules.ts | 3 + tsconfig.json | 2 + yarn.lock | 184 +++++++++++++++++- 22 files changed, 827 insertions(+), 51 deletions(-) create mode 100644 modules/arrow/README.md create mode 100644 modules/arrow/package.json create mode 100644 modules/arrow/src/arrow/analyze-arrow-table.ts create mode 100644 modules/arrow/src/arrow/arrow-column-info.ts create mode 100644 modules/arrow/src/arrow/arrow-paths.ts create mode 100644 modules/arrow/src/arrow/arrow-types.ts create mode 100644 modules/arrow/src/index.ts create mode 100644 modules/arrow/test/arrow/analyze-arrow-table.spec.ts create mode 100644 modules/arrow/test/arrow/arrow-paths.spec.ts create mode 100644 modules/arrow/test/arrow/get-arrow-data.spec.ts create mode 100644 modules/arrow/test/data/arrow/make-arrow-tables.ts create mode 100644 modules/arrow/test/index.ts create mode 100644 modules/arrow/tsconfig.json create mode 100644 modules/engine/README.md diff --git a/modules/arrow/README.md b/modules/arrow/README.md new file mode 100644 index 0000000000..f0c92d3be8 --- /dev/null +++ b/modules/arrow/README.md @@ -0,0 +1,5 @@ +# @luma.gl/arrow + +This is Apache Arrow utilities for luma.gl. + +See [luma.gl](http://luma.gl) for documentation. diff --git a/modules/arrow/package.json b/modules/arrow/package.json new file mode 100644 index 0000000000..b4666d10f5 --- /dev/null +++ b/modules/arrow/package.json @@ -0,0 +1,49 @@ +{ + "private": true, + "name": "@luma.gl/arrow", + "description": "luma.gl Apache Arrow bindings", + "version": "9.2.0-alpha.0", + "license": "MIT", + "type": "module", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/visgl/luma.gl" + }, + "keywords": [ + "webgl", + "visualization", + "animation", + "3d" + ], + "types": "dist/index.d.ts", + "main": "dist/index.cjs", + "module": "dist/index.js", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.cjs", + "types": "./dist/index.d.ts" + } + }, + "files": [ + "src", + "dist", + "dist.min.js", + "README.md" + ], + "sideEffects": false, + "scripts": { + "build-minified-bundle": "ocular-bundle ./bundle.ts --output=dist/dist.min.js", + "build-dev-bundle": "ocular-bundle ./bundle.ts --output=dist/dist.dev.js --env=dev", + "prepublishOnly": "npm run build-minified-bundle && npm run build-dev-bundle" + }, + "dependencies": { + "@luma.gl/core": "9.2.0-alpha.0", + "@math.gl/polygon": "^4.1.0", + "apache-arrow": "^17.0.0" + }, + "gitHead": "c636c34b8f1581eed163e94543a8eb1f4382ba8e" +} diff --git a/modules/arrow/src/arrow/analyze-arrow-table.ts b/modules/arrow/src/arrow/analyze-arrow-table.ts new file mode 100644 index 0000000000..ca88dd42b0 --- /dev/null +++ b/modules/arrow/src/arrow/analyze-arrow-table.ts @@ -0,0 +1,22 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import * as arrow from 'apache-arrow'; +import {getArrowPaths} from './arrow-paths'; +import {ArrowColumnInfo} from './arrow-types'; +import {getArrowColumnInfo} from './arrow-column-info'; + +export function analyzeArrowTable(arrowTable: arrow.Table): Record { + const paths = getArrowPaths(arrowTable); + const columnInfos: Record = {}; + + for (const path of paths) { + const columnInfo = getArrowColumnInfo(arrowTable, path); + if (columnInfo) { + columnInfos[path] = columnInfo; + } + } + + return columnInfos; +} diff --git a/modules/arrow/src/arrow/arrow-column-info.ts b/modules/arrow/src/arrow/arrow-column-info.ts new file mode 100644 index 0000000000..e9db444326 --- /dev/null +++ b/modules/arrow/src/arrow/arrow-column-info.ts @@ -0,0 +1,88 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors +import * as arrow from 'apache-arrow'; +import { + AttributeArrowType, + NumericArrowType, + ArrowColumnInfo, + isNumericArrowType, + isInstanceArrowType, + // isVertexArrowType, + getSignedShaderType +} from './arrow-types'; +import {getArrowVectorByPath} from './arrow-paths'; + +/** Extracts info from columns that can be used as GPU data sources */ +export function getArrowColumnInfo(arrowTable: arrow.Table, path: string): ArrowColumnInfo | null { + const vector = getArrowVectorByPath(arrowTable, path); + if (isInstanceArrowType(vector.type)) { + return getInstanceColumnInfo(vector); + } + // if (isVertexArrowType(vector.type)) { + // return getVertexColumnInfo(vector); + // } + return null; +} + +/** Extracts info from columns that can be used with GPU instanced attributes */ +export function getInstanceColumnInfo(vector: arrow.Vector): ArrowColumnInfo { + let components: 1 | 2 | 3 | 4 = 1; + + let dataVector = vector as arrow.Vector; + if (arrow.DataType.isFixedSizeList(vector.type)) { + dataVector = vector.getChild(0)!; + if (vector.type.listSize < 1 || vector.type.listSize > 4) { + throw new Error('Attribute column fixed list size must be between 1 and 4'); + } + components = vector.type.listSize as 1 | 2 | 3 | 4; + } + + if (!isNumericArrowType(dataVector.type)) { + throw new Error('Attribute column must be numeric or fixed list of numeric'); + } + + const signedDataType = getSignedShaderType(dataVector.type, components); + + const columnInfo: ArrowColumnInfo = { + // data: dataVector.data, + signedDataType, + components, + stepMode: 'instance', + values: [], + offsets: [] + }; + + for (const data of dataVector.data) { + columnInfo.values.push(data.values); + } + return columnInfo; +} + +/** Extracts info from columns that can be used with GPU vertex attributes * +export function getVertexColumnInfo(vector: arrow.Vector): MeshData[] { + if (!arrow.DataType.isList(vector.type)) { + throw new Error('mesh data must be an Arrow list'); + } + + for (const data of vector.data) { + const offsets = data.valueOffsets; + + if (arrow.DataType.isFixedSizeList(vector.type)) { + const dataVector = vector.getChild(0)!; + const getArrowColumnInfo + const dataVectorType = dataVector.type; + if (isNumericArrowType(dataVectorType)) { + return { + data: dataVector.data, + values: dataVector.data.values, + size: vector.type.listSize, + type: getAttributeShaderType(dataVectorType) + }; + } + const size = dataVector; + return vector.getChild(0)!.data; + } + return vector.data; +} +*/ diff --git a/modules/arrow/src/arrow/arrow-paths.ts b/modules/arrow/src/arrow/arrow-paths.ts new file mode 100644 index 0000000000..0f04013b8a --- /dev/null +++ b/modules/arrow/src/arrow/arrow-paths.ts @@ -0,0 +1,115 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import * as arrow from 'apache-arrow'; + +export function getArrowPaths( + arrowObject: arrow.Data | arrow.Table | arrow.RecordBatch | arrow.Vector +): string[] { + const data = getArrowDataArray(arrowObject)[0]; + return getArrowPathsRecursive(data, []); +} + +export function getArrowPathsRecursive(arrowData: arrow.Data, currentPath: string[]): string[] { + if (!arrow.DataType.isStruct(arrowData.type)) { + return [currentPath.join('.')]; + } + + const fields = arrowData.type.children; + const nestedPaths: any[] = []; + for (let fieldIndex = 0; fieldIndex < fields.length; fieldIndex++) { + const field = fields[fieldIndex]; + const fieldData = arrowData.children[fieldIndex]; + const fieldPath = [...currentPath, field.name]; + const paths = getArrowPathsRecursive(fieldData, fieldPath); + nestedPaths.push(...paths); + } + + return nestedPaths; +} + +export function getArrowDataByPath( + arrowObject: arrow.Data | arrow.Table | arrow.RecordBatch | arrow.Vector, + columnPath: string +): arrow.Data { + const data = getArrowDataArray(arrowObject)[0]; + + const path = decomposePath(columnPath); + let nestedData = data; + for (const key of path) { + if (!arrow.DataType.isStruct(nestedData.type)) { + throw new Error( + `Arrow table nested column is a not a struct: '${key} in '${path.join('.')}'` + ); + } + const fields = nestedData.type.children; + const indexByField = fields.findIndex(field => field.name === key); + if (indexByField === -1) { + throw new Error( + `Arrow table schema does not contain nested column '${key} in '${path.join('.')}'` + ); + } + + nestedData = nestedData.children[indexByField]; + } + + // Check that we resolved all the intermediate structs + if (arrow.DataType.isStruct(nestedData.type)) { + throw new Error(`Arrow table nested column '${path.join('.')}' is a struct`); + } + + return nestedData; +} + +export function getArrowVectorByPath(arrowTable: arrow.Table, columnPath: string): arrow.Vector { + // Make a temporary vector from the top level struct data. + const vector = arrow.makeVector(arrowTable.data); + + const path = decomposePath(columnPath); + let nestedVector = vector; + for (const key of path) { + if (!arrow.DataType.isStruct(nestedVector.type)) { + throw new Error( + `Arrow table nested column is a not a struct: '${key} in '${path.join('.')}'` + ); + } + const fields = nestedVector.type.children; + const indexByField = fields.findIndex(field => field.name === key); + if (indexByField === -1) { + throw new Error( + `Arrow table schema does not contain nested column '${key} in '${path.join('.')}'` + ); + } + + nestedVector = nestedVector.getChildAt(indexByField)!; + } + + // Check that we resolved all the intermediate structs + if (arrow.DataType.isStruct(nestedVector.type)) { + throw new Error(`Arrow table nested column '${path.join('.')}' is a struct`); + } + + return nestedVector; +} + +/** Get a data object from an arrow object */ +export function getArrowDataArray( + arrowObject: arrow.Data | arrow.Table | arrow.RecordBatch | arrow.Vector +): arrow.Data[] { + if (arrowObject instanceof arrow.Table) { + return arrowObject.data; + } else if (arrowObject instanceof arrow.RecordBatch) { + return [arrowObject.data]; + } else if (arrowObject instanceof arrow.Vector) { + // @ts-expect-error for some reason read-only in this context + return arrowObject.data; + } + return [arrowObject]; +} + +// HELPER FUNCTIONS + +function decomposePath(path: string): string[] { + return path.split('.'); +} diff --git a/modules/arrow/src/arrow/arrow-types.ts b/modules/arrow/src/arrow/arrow-types.ts new file mode 100644 index 0000000000..351b0d89d4 --- /dev/null +++ b/modules/arrow/src/arrow/arrow-types.ts @@ -0,0 +1,73 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {SignedDataType, BigTypedArray} from '@luma.gl/core'; +import * as arrow from 'apache-arrow'; + +export type NumericArrowType = arrow.Int | arrow.Float; + +/** An instance attribute-compatible column - has 1-4 (fixed) numeric values per row */ +export type AttributeArrowType = NumericArrowType | arrow.FixedSizeList; + +/** A non-instance attribute compatible column - has a list of 1-4 (fixed) numeric values per row */ +export type MeshArrowType = arrow.List>; + +/** Extracted information required to populate a mesh */ +export type ArrowColumnInfo = { + stepMode: 'instance' | 'vertex'; + signedDataType: SignedDataType; + components: 1 | 2 | 3 | 4; + values: BigTypedArray[]; + offsets: Uint32Array[][]; +}; + +export function isNumericArrowType(type: arrow.DataType): type is arrow.Int | arrow.Float { + return arrow.DataType.isFloat(type) || arrow.DataType.isInt(type); +} + +/** Instance = One "vec1-vec4 value" per step */ +export function isInstanceArrowType(type: arrow.DataType): type is AttributeArrowType { + return ( + isNumericArrowType(type) || + (arrow.DataType.isFixedSizeList(type) && isNumericArrowType(type.children[0].type)) + // TODO - check listSize? + ); +} + +/** Vertex = Multiple "vec1-vec4 values" per step */ +export function isVertexArrowType(type: arrow.DataType): type is MeshArrowType { + return arrow.DataType.isList(type) && isInstanceArrowType(type.children[0].type); +} + +/** Get the luma.gl signed shader type corresponding to an Apache Arrow type */ +export function getSignedShaderType( + arrowType: NumericArrowType, + size: 1 | 2 | 3 | 4 +): SignedDataType { + if (arrow.DataType.isInt(arrowType)) { + switch (arrowType.bitWidth) { + case 8: + return arrowType.isSigned ? 'sint8' : 'uint8'; + case 16: + return arrowType.isSigned ? 'sint16' : 'uint16'; + case 32: + return arrowType.isSigned ? 'sint32' : 'uint32'; + case 64: + throw new Error('64-bit integers are not supported in shaders'); + } + } + + if (arrow.DataType.isFloat(arrowType)) { + switch (arrowType.precision) { + case arrow.Precision.HALF: + return 'float16'; + case arrow.Precision.SINGLE: + return 'float32'; + case arrow.Precision.DOUBLE: + throw new Error('Double precision floats are not supported in shaders'); + } + } + + throw new Error(`Unsupported arrow type ${arrowType}`); +} diff --git a/modules/arrow/src/index.ts b/modules/arrow/src/index.ts new file mode 100644 index 0000000000..b18d67c622 --- /dev/null +++ b/modules/arrow/src/index.ts @@ -0,0 +1,16 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +export type {NumericArrowType, ArrowColumnInfo} from './arrow/arrow-types'; +export { + isNumericArrowType + // isInstanceArrowType, + // isVertexArrowType, +} from './arrow/arrow-types'; + +export {getArrowPaths, getArrowDataByPath, getArrowVectorByPath} from './arrow/arrow-paths'; + +export {getArrowColumnInfo} from './arrow/arrow-column-info'; + +export {analyzeArrowTable} from './arrow/analyze-arrow-table'; diff --git a/modules/arrow/test/arrow/analyze-arrow-table.spec.ts b/modules/arrow/test/arrow/analyze-arrow-table.spec.ts new file mode 100644 index 0000000000..fd53613643 --- /dev/null +++ b/modules/arrow/test/arrow/analyze-arrow-table.spec.ts @@ -0,0 +1,21 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import test from 'tape-promise/tape'; +import {ARROW_TABLES} from '@luma.gl/arrow/test/data/arrow/make-arrow-tables'; +import {analyzeArrowTable} from '@luma.gl/arrow'; + +test('getArrowDataByPath', async t => { + const {simpleTable} = ARROW_TABLES; + let tableColumns = analyzeArrowTable(simpleTable); + t.ok(tableColumns, 'extracted info from simple table'); + t.comment(JSON.stringify(tableColumns)); + + const {nestedTable} = ARROW_TABLES; + tableColumns = analyzeArrowTable(nestedTable); + t.ok(tableColumns, 'extracted info from nested table'); + t.comment(JSON.stringify(tableColumns)); + + t.end(); +}); diff --git a/modules/arrow/test/arrow/arrow-paths.spec.ts b/modules/arrow/test/arrow/arrow-paths.spec.ts new file mode 100644 index 0000000000..71fe6853be --- /dev/null +++ b/modules/arrow/test/arrow/arrow-paths.spec.ts @@ -0,0 +1,33 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import test from 'tape-promise/tape'; +import {ARROW_TABLES} from '@luma.gl/arrow/test/data/arrow/make-arrow-tables'; + +import {getArrowPaths, getArrowDataByPath} from '@luma.gl/arrow'; +import * as arrow from 'apache-arrow'; + +test('getArrowTablePaths', async t => { + const {simpleTable} = ARROW_TABLES; + let paths = getArrowPaths(simpleTable); + t.deepEqual(paths, ['age', 'height'], 'got correct paths from simple table'); + + const {nestedTable} = ARROW_TABLES; + paths = getArrowPaths(nestedTable); + t.deepEqual(paths, ['data.age', 'data.height'], 'got correct paths from nested table'); + + t.end(); +}); + +test('getArrowDataByPath', async t => { + const {simpleTable} = ARROW_TABLES; + const ageData = getArrowDataByPath(simpleTable, 'age'); + t.equal(ageData.typeId, arrow.Type.Int, 'extracted age from table struct'); + + const {nestedTable} = ARROW_TABLES; + const heightData = getArrowDataByPath(nestedTable, 'data.height'); + t.equal(heightData.type.typeId, arrow.Type.Float, 'extracted age from nested table struct'); + + t.end(); +}); diff --git a/modules/arrow/test/arrow/get-arrow-data.spec.ts b/modules/arrow/test/arrow/get-arrow-data.spec.ts new file mode 100644 index 0000000000..72258a0e80 --- /dev/null +++ b/modules/arrow/test/arrow/get-arrow-data.spec.ts @@ -0,0 +1,29 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +// import test from 'tape-promise/tape'; +// import * as arrow from 'apache-arrow'; +// import {getArrowDataByPath} from '@luma.gl/arrow'; +// import { expandArrayToCoords } from "../src/utils.js"; +// import { arraysEqual } from "./utils.js"; + +// test('getArrowDataByPath', async t => { +// const nestedTable = makeNestedArrowTable(); +// t.ok(nestedTable, 'nestedTable created'); + +// const ageData = getArrowDataByPath(nestedTable, 'age'); +// t.equal(ageData, 'extracted age from struct'); + +// const deeplyNestedTable = makeDeeplyNestedArrowTable('data'); +// t.ok(deeplyNestedTable, 'deeplyNestedTable created'); + +// const heightData = getArrowDataByPath(deeplyNestedTable, 'data.age'); +// t.equal( +// heightData.type.typeId, +// new arrow.Float32().typeId, +// 'extracted age from deeply nested struct' +// ); + +// t.end(); +// }); diff --git a/modules/arrow/test/data/arrow/make-arrow-tables.ts b/modules/arrow/test/data/arrow/make-arrow-tables.ts new file mode 100644 index 0000000000..0d05b0e42a --- /dev/null +++ b/modules/arrow/test/data/arrow/make-arrow-tables.ts @@ -0,0 +1,64 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import * as arrow from 'apache-arrow'; + +export const ARROW_TABLES = { + simpleTable: makeSimpleArrowTable(), + nestedTable: makeNestedArrowTable('data') +} as const satisfies Record; + +export function makeSimpleArrowTable() { + const structSchema = new arrow.Schema([ + new arrow.Field('age', new arrow.Int32()), + new arrow.Field('height', new arrow.Float32()) + ]); + + const ageData = arrow.makeData({ + type: new arrow.Int32(), + length: 3, + nullCount: 0, + nullBitmap: null, + data: new Int32Array([25, 30, 35]) + }); + + const heightData = arrow.makeData({ + type: new arrow.Float32(), + length: 3, + nullCount: 0, + nullBitmap: null, + data: new Float32Array([25, 30, 35]) + }); + + // Step 3: Use makeData to create the Struct data + const structData = arrow.makeData({ + type: new arrow.Struct(structSchema.fields), + length: 3, + nullCount: 0, + nullBitmap: null, + children: [ageData, heightData] + }); + + const recordBatch = new arrow.RecordBatch(structSchema, structData); + return new arrow.Table([recordBatch]); +} + +export function makeNestedArrowTable(fieldName: string) { + const nestedTable = makeSimpleArrowTable(); + + const innerStructData = nestedTable.batches[0].data; + + const structSchema = new arrow.Schema([new arrow.Field(fieldName, innerStructData.type)]); + + const structData = arrow.makeData({ + type: new arrow.Struct(structSchema.fields), + length: 3, + nullCount: 0, + nullBitmap: null, + children: [innerStructData] + }); + + const recordBatch = new arrow.RecordBatch(structSchema, structData); + return new arrow.Table([recordBatch]); +} diff --git a/modules/arrow/test/index.ts b/modules/arrow/test/index.ts new file mode 100644 index 0000000000..7b851e44c5 --- /dev/null +++ b/modules/arrow/test/index.ts @@ -0,0 +1,7 @@ +// luma.gl +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import './arrow/arrow-paths.spec'; +import './arrow/get-arrow-data.spec'; +import './arrow/analyze-arrow-table.spec'; diff --git a/modules/arrow/tsconfig.json b/modules/arrow/tsconfig.json new file mode 100644 index 0000000000..c41b9ffade --- /dev/null +++ b/modules/arrow/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*"], + "exclude": ["node_modules"], + "compilerOptions": { + "composite": true, + "rootDir": "src", + "outDir": "dist" + }, + "references": [ + {"path": "../core"}, + {"path": "../engine"} + ] +} diff --git a/modules/core/src/index.ts b/modules/core/src/index.ts index ca8d0a62a5..1dddaf6901 100644 --- a/modules/core/src/index.ts +++ b/modules/core/src/index.ts @@ -123,9 +123,18 @@ export type { export type {UniformValue} from './adapter/types/uniforms'; +// TYPED ARRAY TYPES + +export type { + NumberArray, + TypedArray, + TypedArrayConstructor, + BigTypedArray, + BigTypedArrayConstructor +} from './types'; + // GPU TYPE UTILS - GPU MEMORY LAYOUT TYPES - EXTERNAL -export type {NumberArray, TypedArray, TypedArrayConstructor} from './types'; export type {PrimitiveDataType, SignedDataType, NormalizedDataType} from './shadertypes/data-types'; export type {AttributeShaderType, VariableShaderType} from './shadertypes/shader-types'; export type {VertexFormat} from './shadertypes/vertex-formats'; @@ -143,15 +152,17 @@ export type { export { getDataTypeInfo, getDataTypeFromTypedArray, - getTypedArrayFromDataType + getTypedArrayFromDataType, + makeNormalizedDataType } from './shadertypes/utils/decode-data-types'; export { getVariableShaderTypeInfo, getAttributeShaderTypeInfo } from './shadertypes/utils/decode-shader-types'; export { + getVertexFormatInfo, getVertexFormatFromAttribute, - getVertexFormatInfo + makeVertexFormat } from './shadertypes/utils/decode-vertex-format'; export { getTextureFormatInfo, diff --git a/modules/core/src/shadertypes/utils/decode-data-types.ts b/modules/core/src/shadertypes/utils/decode-data-types.ts index 8599c9d7c3..bb62fc7780 100644 --- a/modules/core/src/shadertypes/utils/decode-data-types.ts +++ b/modules/core/src/shadertypes/utils/decode-data-types.ts @@ -25,6 +25,24 @@ export function getDataTypeInfo(type: NormalizedDataType): DataTypeInfo { }; } +/** Build a vertex format from a signed data type and a component */ +export function makeNormalizedDataType(signedDataType: SignedDataType): NormalizedDataType { + const dataType: NormalizedDataType = signedDataType; + + switch (dataType) { + case 'uint8': + return 'unorm8'; + case 'sint8': + return 'snorm8'; + case 'uint16': + return 'unorm16'; + case 'sint16': + return 'snorm16'; + default: + return dataType; + } +} + /** Align offset to 1, 2 or 4 elements (4, 8 or 16 bytes) */ export function alignTo(size: number, count: number): number { // prettier-ignore diff --git a/modules/core/src/shadertypes/utils/decode-shader-types.ts b/modules/core/src/shadertypes/utils/decode-shader-types.ts index 631e3378e8..d3d357f3d1 100644 --- a/modules/core/src/shadertypes/utils/decode-shader-types.ts +++ b/modules/core/src/shadertypes/utils/decode-shader-types.ts @@ -38,6 +38,13 @@ export function getAttributeShaderTypeInfo( }; } +export function makeShaderAttributeType( + primitiveType: PrimitiveDataType, + components: 1 | 2 | 3 | 4 +): AttributeShaderType { + return components === 1 ? primitiveType : `vec${components}<${primitiveType}>`; +} + export function resolveAttributeShaderTypeAlias( alias: AttributeShaderTypeAlias | AttributeShaderType ): AttributeShaderType { diff --git a/modules/core/src/shadertypes/utils/decode-vertex-format.ts b/modules/core/src/shadertypes/utils/decode-vertex-format.ts index e4a4423636..5221252d51 100644 --- a/modules/core/src/shadertypes/utils/decode-vertex-format.ts +++ b/modules/core/src/shadertypes/utils/decode-vertex-format.ts @@ -3,9 +3,13 @@ // Copyright (c) vis.gl contributors import type {TypedArray} from '../../types'; -import type {NormalizedDataType, PrimitiveDataType} from '../data-types'; +import type {NormalizedDataType, PrimitiveDataType, SignedDataType} from '../data-types'; import type {VertexFormat, VertexFormatInfo} from '../vertex-formats'; -import {getDataTypeInfo, getDataTypeFromTypedArray} from './decode-data-types'; +import { + getDataTypeInfo, + getDataTypeFromTypedArray, + makeNormalizedDataType +} from './decode-data-types'; /** * Decodes a vertex format, returning type, components, byte length and flags (integer, signed, normalized) @@ -37,6 +41,48 @@ export function getVertexFormatInfo(format: VertexFormat): VertexFormatInfo { return result; } +/** Build a vertex format from a signed data type and a component */ +export function makeVertexFormat( + signedDataType: SignedDataType, + components: 1 | 2 | 3 | 4, + normalized?: boolean +): VertexFormat { + const dataType: NormalizedDataType = normalized + ? makeNormalizedDataType(signedDataType) + : signedDataType; + + switch (dataType) { + // TODO - Special cases for WebGL (not supported on WebGPU), overrides the check below + case 'unorm8': + if (components === 1) { + return 'unorm8-webgl'; + } + if (components === 3) { + return 'unorm8x3-webgl'; + } + return `${dataType}x${components}`; + + case 'snorm8': + case 'uint8': + case 'sint8': + // WebGPU 8 bit formats must be aligned to 16 bit boundaries'); + // fall through + case 'uint16': + case 'sint16': + case 'unorm16': + case 'snorm16': + case 'float16': + // WebGPU 16 bit formats must be aligned to 32 bit boundaries + if (components === 1 || components === 3) { + throw new Error(`size: ${components}`); + } + return `${dataType}x${components}`; + + default: + return components === 1 ? dataType : `${dataType}x${components}`; + } +} + /** Get the vertex format for an attribute with TypedArray and size */ export function getVertexFormatFromAttribute( typedArray: TypedArray, @@ -49,44 +95,7 @@ export function getVertexFormatFromAttribute( const components = size as 1 | 2 | 3 | 4; const signedDataType = getDataTypeFromTypedArray(typedArray); - - let dataType: NormalizedDataType = signedDataType; - - // TODO - Special cases for WebGL (not supported on WebGPU), overrides the check below - if (dataType === 'uint8' && normalized && components === 1) { - return 'unorm8-webgl'; - } - if (dataType === 'uint8' && normalized && components === 3) { - return 'unorm8x3-webgl'; - } - - if (dataType === 'uint8' || dataType === 'sint8') { - if (components === 1 || components === 3) { - // WebGPU 8 bit formats must be aligned to 16 bit boundaries'); - throw new Error(`size: ${size}`); - } - if (normalized) { - dataType = dataType.replace('int', 'norm') as 'unorm8' | 'snorm8'; - } - return `${dataType}x${components}`; - } - if (dataType === 'uint16' || dataType === 'sint16') { - if (components === 1 || components === 3) { - // WebGPU 16 bit formats must be aligned to 32 bit boundaries - throw new Error(`size: ${size}`); - } - if (normalized) { - dataType = dataType.replace('int', 'norm') as 'unorm16' | 'snorm16'; - } - return `${dataType}x${components}`; - } - - if (components === 1 && dataType !== 'float16') { - return dataType; - } - - // @ts-expect-error TODO fix - return `${dataType}x${components}`; + return makeVertexFormat(signedDataType, components, normalized); } /** Return a "default" vertex format for a certain shader data type diff --git a/modules/core/src/types.ts b/modules/core/src/types.ts index 760ed6a5f6..b338ba0efe 100644 --- a/modules/core/src/types.ts +++ b/modules/core/src/types.ts @@ -7,7 +7,14 @@ import {TypedArray, NumberArray} from '@math.gl/types'; export {TypedArray, NumberArray}; -/** TypeScript type covering constructors of any of the typed arrays */ +export type BigTypedArray = TypedArray | BigIntTypedArray; + +/** Keep big int arrays separate as they are still problematic, can't be indexed and don't work well on Safari */ +export type BigIntTypedArray = BigInt64Array | BigUint64Array; + +export type BigIntOrNumberArray = NumberArray | BigIntTypedArray; + +/** TypeScript type covering constructors of any of the typed arrays, except BigInt */ export type TypedArrayConstructor = | Int8ArrayConstructor | Uint8ArrayConstructor @@ -19,7 +26,7 @@ export type TypedArrayConstructor = | Float32ArrayConstructor | Float64ArrayConstructor; -/** Keep big int arrays separate as they are still problematic, can't be indexed and don't work well on Safari */ -export type BigIntTypedArray = BigInt64Array | BigUint64Array; - -export type BigIntOrNumberArray = NumberArray | BigIntTypedArray; +export type BigTypedArrayConstructor = + | TypedArrayConstructor + | BigInt64ArrayConstructor + | BigUint64ArrayConstructor; diff --git a/modules/engine/README.md b/modules/engine/README.md new file mode 100644 index 0000000000..e69276901c --- /dev/null +++ b/modules/engine/README.md @@ -0,0 +1,5 @@ +# @luma.gl/engine + +This is the engine layer of luma.gl + +See [luma.gl](http://luma.gl) for documentation. diff --git a/test/modules.ts b/test/modules.ts index 8b3b143c59..f59adb7334 100644 --- a/test/modules.ts +++ b/test/modules.ts @@ -23,5 +23,8 @@ import '@luma.gl/engine/test'; import '@luma.gl/gltf/test'; // import '@luma.gl/experimental/test'; +// EXPERIMENTAL +import '@luma.gl/arrow/test'; + // DEPRECATED TESTS import '@luma.gl/constants/test'; diff --git a/tsconfig.json b/tsconfig.json index 9ff687f6bf..762bcb2421 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -40,6 +40,8 @@ // END TYPE CHECK SETTINGS "paths": { + "@luma.gl/arrow/*": ["modules/arrow/src/*"], + "@luma.gl/arrow/test/*": ["modules/arrow/test/*"], "@luma.gl/core/*": ["modules/core/src/*"], "@luma.gl/core/test/*": ["modules/core/test/*"], "@luma.gl/constants/*": ["modules/constants/src/*"], diff --git a/yarn.lock b/yarn.lock index a85417fa32..51dc1cfbf1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -986,6 +986,16 @@ __metadata: languageName: node linkType: hard +"@luma.gl/arrow@workspace:modules/arrow": + version: 0.0.0-use.local + resolution: "@luma.gl/arrow@workspace:modules/arrow" + dependencies: + "@luma.gl/core": "npm:9.2.0-alpha.0" + "@math.gl/polygon": "npm:^4.1.0" + apache-arrow: "npm:^17.0.0" + languageName: unknown + linkType: soft + "@luma.gl/constants@npm:^9.0.0": version: 9.0.27 resolution: "@luma.gl/constants@npm:9.0.27" @@ -999,7 +1009,7 @@ __metadata: languageName: unknown linkType: soft -"@luma.gl/core@workspace:modules/core": +"@luma.gl/core@npm:9.2.0-alpha.0, @luma.gl/core@workspace:modules/core": version: 0.0.0-use.local resolution: "@luma.gl/core@workspace:modules/core" dependencies: @@ -1101,7 +1111,7 @@ __metadata: languageName: unknown linkType: soft -"@math.gl/core@npm:^4.0.0, @math.gl/core@npm:^4.1.0": +"@math.gl/core@npm:4.1.0, @math.gl/core@npm:^4.0.0, @math.gl/core@npm:^4.1.0": version: 4.1.0 resolution: "@math.gl/core@npm:4.1.0" dependencies: @@ -1110,6 +1120,15 @@ __metadata: languageName: node linkType: hard +"@math.gl/polygon@npm:^4.1.0": + version: 4.1.0 + resolution: "@math.gl/polygon@npm:4.1.0" + dependencies: + "@math.gl/core": "npm:4.1.0" + checksum: 10c0/0fcfb489c5613ddff6dd0cbea65084e10fa9a3523fb87a36a4fdf10057d12d2a99f1ebd93da6e72b45db0783fb8cd9cff704765473372a8db580faaf50c85ab5 + languageName: node + linkType: hard + "@math.gl/types@npm:4.1.0, @math.gl/types@npm:^4.0.1, @math.gl/types@npm:^4.1.0": version: 4.1.0 resolution: "@math.gl/types@npm:4.1.0" @@ -1775,6 +1794,15 @@ __metadata: languageName: node linkType: hard +"@swc/helpers@npm:^0.5.11": + version: 0.5.13 + resolution: "@swc/helpers@npm:0.5.13" + dependencies: + tslib: "npm:^2.4.0" + checksum: 10c0/b9df578401fc62405da9a6c31e79e447a2fd90f68b25b1daee12f2caf2821991bb89106f0397bc1acb4c4d84a8ce079d04b60b65f534496952e3bf8c9a52f40f + languageName: node + linkType: hard + "@tootallnate/quickjs-emscripten@npm:^0.23.0": version: 0.23.0 resolution: "@tootallnate/quickjs-emscripten@npm:0.23.0" @@ -1836,6 +1864,20 @@ __metadata: languageName: node linkType: hard +"@types/command-line-args@npm:^5.2.3": + version: 5.2.3 + resolution: "@types/command-line-args@npm:5.2.3" + checksum: 10c0/3a9bc58fd26e546391f6369dd28c03d59349dc4ac39eada1a5c39cc3578e02e4aac222615170e0db79b198ffba2af84fdbdda46e08c6edc4da42bc17ea85200f + languageName: node + linkType: hard + +"@types/command-line-usage@npm:^5.0.4": + version: 5.0.4 + resolution: "@types/command-line-usage@npm:5.0.4" + checksum: 10c0/67840ebf4bcfee200c07d978669ad596fe2adc350fd5c19d44ec2248623575d96ec917f513d1d59453f8f57e879133861a4cc41c20045c07f6c959f1fcaac7ad + languageName: node + linkType: hard + "@types/crypto-js@npm:^4.0.2": version: 4.2.2 resolution: "@types/crypto-js@npm:4.2.2" @@ -1896,6 +1938,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^20.13.0": + version: 20.16.11 + resolution: "@types/node@npm:20.16.11" + dependencies: + undici-types: "npm:~6.19.2" + checksum: 10c0/bba43f447c3c80548513954dae174e18132e9149d572c09df4a282772960d33e229d05680fb5364997c03489c22fe377d1dbcd018a3d4ff1cfbcfcdaa594a9c3 + languageName: node + linkType: hard + "@types/normalize-package-data@npm:^2.4.0": version: 2.4.4 resolution: "@types/normalize-package-data@npm:2.4.4" @@ -2342,6 +2393,25 @@ __metadata: languageName: node linkType: hard +"apache-arrow@npm:^17.0.0": + version: 17.0.0 + resolution: "apache-arrow@npm:17.0.0" + dependencies: + "@swc/helpers": "npm:^0.5.11" + "@types/command-line-args": "npm:^5.2.3" + "@types/command-line-usage": "npm:^5.0.4" + "@types/node": "npm:^20.13.0" + command-line-args: "npm:^5.2.1" + command-line-usage: "npm:^7.0.1" + flatbuffers: "npm:^24.3.25" + json-bignum: "npm:^0.0.3" + tslib: "npm:^2.6.2" + bin: + arrow2csv: bin/arrow2csv.cjs + checksum: 10c0/fa705b88a5d76c05122414bf7ba8a46c93f3509584de332eb045c736384cdfe53fc140135abbdeaa6036dfe422d145598b461b89f0b710cdeebbb732e9fb7f58 + languageName: node + linkType: hard + "append-transform@npm:^2.0.0": version: 2.0.0 resolution: "append-transform@npm:2.0.0" @@ -2397,6 +2467,20 @@ __metadata: languageName: node linkType: hard +"array-back@npm:^3.0.1, array-back@npm:^3.1.0": + version: 3.1.0 + resolution: "array-back@npm:3.1.0" + checksum: 10c0/bb1fe86aa8b39c21e73c68c7abf8b05ed939b8951a3b17527217f6a2a84e00e4cfa4fdec823081689c5e216709bf1f214a4f5feeee6726eaff83897fa1a7b8ee + languageName: node + linkType: hard + +"array-back@npm:^6.2.2": + version: 6.2.2 + resolution: "array-back@npm:6.2.2" + checksum: 10c0/c98a6e43b669400f58e2fba478336d5d02aac970566ffae3af0cb9b5585ec3811a1e010c76e34fb809a9762e6822a43a9c9a1b99f2a35f43b11a9e198e782818 + languageName: node + linkType: hard + "array-buffer-byte-length@npm:^1.0.0, array-buffer-byte-length@npm:^1.0.1": version: 1.0.1 resolution: "array-buffer-byte-length@npm:1.0.1" @@ -2938,6 +3022,15 @@ __metadata: languageName: node linkType: hard +"chalk-template@npm:^0.4.0": + version: 0.4.0 + resolution: "chalk-template@npm:0.4.0" + dependencies: + chalk: "npm:^4.1.2" + checksum: 10c0/6a4cb4252966475f0bd3ee1cd8780146e1ba69f445e59c565cab891ac18708c8143515d23e2b0fb7e192574fb7608d429ea5b28f3b7b9507770ad6fccd3467e3 + languageName: node + linkType: hard + "chalk@npm:4.1.0": version: 4.1.0 resolution: "chalk@npm:4.1.0" @@ -3199,6 +3292,30 @@ __metadata: languageName: node linkType: hard +"command-line-args@npm:^5.2.1": + version: 5.2.1 + resolution: "command-line-args@npm:5.2.1" + dependencies: + array-back: "npm:^3.1.0" + find-replace: "npm:^3.0.0" + lodash.camelcase: "npm:^4.3.0" + typical: "npm:^4.0.0" + checksum: 10c0/a4f6a23a1e420441bd1e44dee24efd12d2e49af7efe6e21eb32fca4e843ca3d5501ddebad86a4e9d99aa626dd6dcb64c04a43695388be54e3a803dbc326cc89f + languageName: node + linkType: hard + +"command-line-usage@npm:^7.0.1": + version: 7.0.3 + resolution: "command-line-usage@npm:7.0.3" + dependencies: + array-back: "npm:^6.2.2" + chalk-template: "npm:^0.4.0" + table-layout: "npm:^4.1.0" + typical: "npm:^7.1.1" + checksum: 10c0/444a3e3c6fcbdcb5802de0fd2864ea5aef83eeeb3a825fd24846b996503d4b4140e75aeb2939b3430a06407f3acc02b76b3e08dafb3a3092d22fdcced0ecb0b0 + languageName: node + linkType: hard + "common-ancestor-path@npm:^1.0.1": version: 1.0.1 resolution: "common-ancestor-path@npm:1.0.1" @@ -4780,6 +4897,15 @@ __metadata: languageName: node linkType: hard +"find-replace@npm:^3.0.0": + version: 3.0.0 + resolution: "find-replace@npm:3.0.0" + dependencies: + array-back: "npm:^3.0.1" + checksum: 10c0/fcd1bf7960388c8193c2861bcdc760c18ac14edb4bde062a961915d9a25727b2e8aabf0229e90cc09c753fd557e5a3e5ae61e49cadbe727be89a9e8e49ce7668 + languageName: node + linkType: hard + "find-up@npm:^2.0.0": version: 2.1.0 resolution: "find-up@npm:2.1.0" @@ -4829,6 +4955,13 @@ __metadata: languageName: node linkType: hard +"flatbuffers@npm:^24.3.25": + version: 24.3.25 + resolution: "flatbuffers@npm:24.3.25" + checksum: 10c0/a40a1d46f6d474f1299091970700f36dc5bd86616b71cd99a1b6f521ca33b05f5d7623bd0ffadaa1e10fc63d6663aa3f9595f4916cd379e80c15787257e17a61 + languageName: node + linkType: hard + "flatted@npm:^3.2.9": version: 3.3.1 resolution: "flatted@npm:3.3.1" @@ -6406,6 +6539,13 @@ __metadata: languageName: node linkType: hard +"json-bignum@npm:^0.0.3": + version: 0.0.3 + resolution: "json-bignum@npm:0.0.3" + checksum: 10c0/f9f9312d57a68f72676802fa087da4ed60241d73b6cc0e3fb9f587ca0de7364efb62612a14414ccfbedc0b77ce3c320adca21834a5673c99eb3375aef9f561db + languageName: node + linkType: hard + "json-buffer@npm:3.0.1": version: 3.0.1 resolution: "json-buffer@npm:3.0.1" @@ -6795,6 +6935,13 @@ __metadata: languageName: node linkType: hard +"lodash.camelcase@npm:^4.3.0": + version: 4.3.0 + resolution: "lodash.camelcase@npm:4.3.0" + checksum: 10c0/fcba15d21a458076dd309fce6b1b4bf611d84a0ec252cb92447c948c533ac250b95d2e00955801ebc367e5af5ed288b996d75d37d2035260a937008e14eaf432 + languageName: node + linkType: hard + "lodash.flattendeep@npm:^4.4.0": version: 4.4.0 resolution: "lodash.flattendeep@npm:4.4.0" @@ -9818,6 +9965,16 @@ __metadata: languageName: node linkType: hard +"table-layout@npm:^4.1.0": + version: 4.1.1 + resolution: "table-layout@npm:4.1.1" + dependencies: + array-back: "npm:^6.2.2" + wordwrapjs: "npm:^5.1.0" + checksum: 10c0/26d8e54a55ddb4de447c8f02a8d7fcbb66a9580375e406a3bc7717ab223a413f6dfbded6710f288b3dfd277991813a0bd5a17419a0dc6db54d9a36d883d868dc + languageName: node + linkType: hard + "tap-out@npm:^2.1.0": version: 2.1.0 resolution: "tap-out@npm:2.1.0" @@ -10158,7 +10315,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.0.1, tslib@npm:^2.1.0, tslib@npm:^2.3.0, tslib@npm:^2.4.0": +"tslib@npm:^2.0.1, tslib@npm:^2.1.0, tslib@npm:^2.3.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2": version: 2.7.0 resolution: "tslib@npm:2.7.0" checksum: 10c0/469e1d5bf1af585742128827000711efa61010b699cb040ab1800bcd3ccdd37f63ec30642c9e07c4439c1db6e46345582614275daca3e0f4abae29b0083f04a6 @@ -10347,6 +10504,20 @@ __metadata: languageName: node linkType: hard +"typical@npm:^4.0.0": + version: 4.0.0 + resolution: "typical@npm:4.0.0" + checksum: 10c0/f300b198fb9fe743859b75ec761d53c382723dc178bbce4957d9cb754f2878a44ce17dc0b6a5156c52be1065449271f63754ba594dac225b80ce3aa39f9241ed + languageName: node + linkType: hard + +"typical@npm:^7.1.1": + version: 7.2.0 + resolution: "typical@npm:7.2.0" + checksum: 10c0/aa447e761808c9447c3abde370f2bdd2edd031ff68183aac49ac503905155e66a9f47e1462ac6fa411f76b22920c4d403f948f49d984ebf52d019fa590034963 + languageName: node + linkType: hard + "uglify-js@npm:^3.1.4": version: 3.19.3 resolution: "uglify-js@npm:3.19.3" @@ -10769,6 +10940,13 @@ __metadata: languageName: node linkType: hard +"wordwrapjs@npm:^5.1.0": + version: 5.1.0 + resolution: "wordwrapjs@npm:5.1.0" + checksum: 10c0/e147162f139eb8c05257729fde586f5422a2d242aa8f027b5fa5adead1b571b455d0690a15c73aeaa31c93ba96864caa06d84ebdb2c32a0890602ab86a7568d1 + languageName: node + linkType: hard + "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0, wrap-ansi@npm:^7.0.0": version: 7.0.0 resolution: "wrap-ansi@npm:7.0.0"