From efe910655407b12fb356894b44a6be369541544b Mon Sep 17 00:00:00 2001 From: Matthew Nibecker Date: Wed, 10 Jan 2024 11:44:29 -0700 Subject: [PATCH 01/43] wip --- apps/zui/package.json | 1 + apps/zui/src/js/models/program.ts | 8 +- apps/zui/tsconfig.json | 1 + packages/zed-wasm/go.mod | 45 +- packages/zed-wasm/go.sum | 199 ++-- packages/zed-wasm/index.html | 5 +- packages/zed-wasm/lib/wasm_exec.js | 1208 +++++++++++------------ packages/zed-wasm/lib/wasm_exec_node.js | 59 +- packages/zed-wasm/main.go | 22 +- packages/zed-wasm/package.json | 6 +- packages/zed-wasm/project.json | 24 +- packages/zed-wasm/src/browser.ts | 39 + packages/zed-wasm/src/index.ts | 41 +- packages/zed-wasm/src/input.ts | 25 + packages/zed-wasm/test/parse.test.js | 13 + packages/zed-wasm/tsconfig.json | 7 +- packages/zed-wasm/tsconfig.lib.json | 2 +- yarn.lock | 3 +- 18 files changed, 810 insertions(+), 898 deletions(-) create mode 100644 packages/zed-wasm/src/browser.ts create mode 100644 packages/zed-wasm/src/input.ts create mode 100644 packages/zed-wasm/test/parse.test.js diff --git a/apps/zui/package.json b/apps/zui/package.json index cac7d7c72c..0d1c98eaa2 100644 --- a/apps/zui/package.json +++ b/apps/zui/package.json @@ -39,6 +39,7 @@ "@babel/core": "^7.17.9", "@brimdata/zed-js": "workspace:*", "@brimdata/zed-node": "workspace:*", + "@brimdata/zed-wasm": "workspace:*", "@js-joda/core": "^3.2.0", "@monaco-editor/react": "^4.5.1", "@reduxjs/toolkit": "^1.9.3", diff --git a/apps/zui/src/js/models/program.ts b/apps/zui/src/js/models/program.ts index bc25202a8e..41fcab7484 100644 --- a/apps/zui/src/js/models/program.ts +++ b/apps/zui/src/js/models/program.ts @@ -1,5 +1,5 @@ import * as zed from "@brimdata/zed-js" -import {parse as parseAst} from "zed/compiler/parser/parser" +import {parse as parseAst} from "@brimdata/zed-wasm" import {isEmpty, last} from "lodash" import {trim} from "../lib/Str" import ast, {ANALYTIC_PROCS} from "./ast" @@ -64,10 +64,12 @@ export default function (p = "") { return this }, - ast() { + async ast() { let tree try { - tree = parseAst(p) + let res = await parseAst(p) + if (res.error) throw res.error + tree = res.ast } catch (error) { tree = {error} } diff --git a/apps/zui/tsconfig.json b/apps/zui/tsconfig.json index 8e0d967770..5feb82e22a 100644 --- a/apps/zui/tsconfig.json +++ b/apps/zui/tsconfig.json @@ -21,6 +21,7 @@ "paths": { "@brimdata/zed-js": ["../../packages/zed-js/src/index.ts"], "@brimdata/zed-node": ["../../packages/zed-node/src/index.ts"], + "@brimdata/zed-wasm": ["../../packages/zed-wasm/src/index.ts"], "zui-test-data": ["../../packages/zui-test-data/src/index.ts"], } }, diff --git a/packages/zed-wasm/go.mod b/packages/zed-wasm/go.mod index 5d4ba66af1..f539f78386 100644 --- a/packages/zed-wasm/go.mod +++ b/packages/zed-wasm/go.mod @@ -3,7 +3,7 @@ module zqjs go 1.21 require ( - github.com/brimdata/zed v1.6.0 + github.com/brimdata/zed v1.12.0 github.com/teamortix/golang-wasm/wasm v0.0.0-20230308073412-915550b3b9ac ) @@ -11,43 +11,42 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect - github.com/andybalholm/brotli v1.0.4 // indirect - github.com/apache/arrow/go/v11 v11.0.0-20221214174703-0dfec8e98f4f // indirect - github.com/apache/thrift v0.16.0 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/apache/arrow/go/v14 v14.0.0 // indirect + github.com/apache/thrift v0.17.0 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect github.com/aws/aws-sdk-go v1.36.17 // indirect github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect - github.com/fraugster/parquet-go v0.10.1-0.20220222153523-e6b70a8a7212 // indirect - github.com/goccy/go-json v0.9.11 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/text v0.2.0 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect - github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/segmentio/ksuid v1.0.2 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.23.0 // indirect - golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.2.0 // indirect - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect - google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect - google.golang.org/grpc v1.49.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sync v0.4.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/grpc v1.58.2 // indirect + google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/packages/zed-wasm/go.sum b/packages/zed-wasm/go.sum index 8e61eed971..9330966b06 100644 --- a/packages/zed-wasm/go.sum +++ b/packages/zed-wasm/go.sum @@ -1,5 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -7,32 +5,25 @@ github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRB github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/apache/arrow/go/v11 v11.0.0-20221214174703-0dfec8e98f4f h1:QUx+UUDqXqbmYrTUhaLVv6UZFQ13DT3uyK8JvweSvO4= -github.com/apache/arrow/go/v11 v11.0.0-20221214174703-0dfec8e98f4f/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= -github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/apache/arrow/go/v14 v14.0.0 h1:NXfgmvrHAWSzPO1YNjDhO9VwYrUQI/kRvzy5dJuCIaY= +github.com/apache/arrow/go/v14 v14.0.0/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= +github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= +github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.36.17 h1:8zTvseyGhgs3uQAzkgnFy7dvTo+ZnZLYmrhnopFxYME= github.com/aws/aws-sdk-go v1.36.17/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f h1:y06x6vGnFYfXUoVMbrcP1Uzpj4JG01eB5vRps9G8agM= github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f/go.mod h1:2stgcRjl6QmW+gU2h5E7BQXg4HU0gzxKWDuT5HviN9s= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/brimdata/zed v1.6.0 h1:/XB4xZZWAb9qbax0Fx0YuMAYfc7r/5EVvDfNgY8yOHo= -github.com/brimdata/zed v1.6.0/go.mod h1:WVzlx/gazFjLgHTCPsxYlU8eGxryCpii6nSVT/cJkJY= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/brimdata/zed v1.12.0 h1:JzDjjcdEI6GICUG07d2B3tQnRWcfUav9bglXzKnVXpY= +github.com/brimdata/zed v1.12.0/go.mod h1:bD4XpGLwJ2ZHUDBVtPuY+KCF5ogemaOXW2NTrVnRuaw= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -41,42 +32,24 @@ github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsY github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fraugster/parquet-go v0.10.1-0.20220222153523-e6b70a8a7212 h1:u7X3aZRlWSm18x0EysX9szRULhH7QYQv7UkxW1yHbik= -github.com/fraugster/parquet-go v0.10.1-0.20220222153523-e6b70a8a7212/go.mod h1:dGzUxdNqXsAijatByVgbAWVPlFirnhknQbdazcUIjY0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= -github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.7.6/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -84,59 +57,40 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/segmentio/ksuid v1.0.2 h1:9yBfKyw4ECGTdALaF09Snw3sLJmYIX6AbPJrAy6MrDc= github.com/segmentio/ksuid v1.0.2/go.mod h1:BXuJDr2byAiHuQaQtSKoXh1J0YmUDurywOXgB2w+OSU= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/teamortix/golang-wasm/wasm v0.0.0-20230308073412-915550b3b9ac h1:2K3l7bvK0s5x+Qkv2paUqlZm+FI5RzoJ/oQLUwWGYvk= github.com/teamortix/golang-wasm/wasm v0.0.0-20230308073412-915550b3b9ac/go.mod h1:nskvTyoGIaAsC+664SkRitVI1ft6dm1xerCr50YZsnY= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -149,88 +103,47 @@ go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= -golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= -gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 h1:PDIOdWxZ8eRizhKa1AAvY53xsvLB1cWorMjslvY3VA8= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -238,5 +151,3 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/packages/zed-wasm/index.html b/packages/zed-wasm/index.html index c5866f1637..a6aa46b708 100644 --- a/packages/zed-wasm/index.html +++ b/packages/zed-wasm/index.html @@ -43,9 +43,12 @@

Unit Tests

+ + diff --git a/packages/zed-wasm/lib/wasm_exec.js b/packages/zed-wasm/lib/wasm_exec.js index 2128077b9d..bc6f210242 100644 --- a/packages/zed-wasm/lib/wasm_exec.js +++ b/packages/zed-wasm/lib/wasm_exec.js @@ -2,660 +2,560 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -'use strict'; +"use strict"; (() => { - const enosys = () => { - const err = new Error('not implemented'); - err.code = 'ENOSYS'; - return err; - }; - - if (!globalThis.fs) { - let outputBuf = ''; - globalThis.fs = { - constants: { - O_WRONLY: -1, - O_RDWR: -1, - O_CREAT: -1, - O_TRUNC: -1, - O_APPEND: -1, - O_EXCL: -1, - }, // unused - writeSync(fd, buf) { - outputBuf += decoder.decode(buf); - const nl = outputBuf.lastIndexOf('\n'); - if (nl != -1) { - console.log(outputBuf.substr(0, nl)); - outputBuf = outputBuf.substr(nl + 1); - } - return buf.length; - }, - write(fd, buf, offset, length, position, callback) { - if (offset !== 0 || length !== buf.length || position !== null) { - callback(enosys()); - return; - } - const n = this.writeSync(fd, buf); - callback(null, n); - }, - chmod(path, mode, callback) { - callback(enosys()); - }, - chown(path, uid, gid, callback) { - callback(enosys()); - }, - close(fd, callback) { - callback(enosys()); - }, - fchmod(fd, mode, callback) { - callback(enosys()); - }, - fchown(fd, uid, gid, callback) { - callback(enosys()); - }, - fstat(fd, callback) { - callback(enosys()); - }, - fsync(fd, callback) { - callback(null); - }, - ftruncate(fd, length, callback) { - callback(enosys()); - }, - lchown(path, uid, gid, callback) { - callback(enosys()); - }, - link(path, link, callback) { - callback(enosys()); - }, - lstat(path, callback) { - callback(enosys()); - }, - mkdir(path, perm, callback) { - callback(enosys()); - }, - open(path, flags, mode, callback) { - callback(enosys()); - }, - read(fd, buffer, offset, length, position, callback) { - callback(enosys()); - }, - readdir(path, callback) { - callback(enosys()); - }, - readlink(path, callback) { - callback(enosys()); - }, - rename(from, to, callback) { - callback(enosys()); - }, - rmdir(path, callback) { - callback(enosys()); - }, - stat(path, callback) { - callback(enosys()); - }, - symlink(path, link, callback) { - callback(enosys()); - }, - truncate(path, length, callback) { - callback(enosys()); - }, - unlink(path, callback) { - callback(enosys()); - }, - utimes(path, atime, mtime, callback) { - callback(enosys()); - }, - }; - } - - if (!globalThis.process) { - globalThis.process = { - getuid() { - return -1; - }, - getgid() { - return -1; - }, - geteuid() { - return -1; - }, - getegid() { - return -1; - }, - getgroups() { - throw enosys(); - }, - pid: -1, - ppid: -1, - umask() { - throw enosys(); - }, - cwd() { - throw enosys(); - }, - chdir() { - throw enosys(); - }, - }; - } - - if (!globalThis.crypto) { - throw new Error( - 'globalThis.crypto is not available, polyfill required (crypto.getRandomValues only)' - ); - } - - if (!globalThis.performance) { - throw new Error( - 'globalThis.performance is not available, polyfill required (performance.now only)' - ); - } - - if (!globalThis.TextEncoder) { - throw new Error( - 'globalThis.TextEncoder is not available, polyfill required' - ); - } - - if (!globalThis.TextDecoder) { - throw new Error( - 'globalThis.TextDecoder is not available, polyfill required' - ); - } - - const encoder = new TextEncoder('utf-8'); - const decoder = new TextDecoder('utf-8'); - - globalThis.Go = class { - constructor() { - this.argv = ['js']; - this.env = {}; - this.exit = (code) => { - if (code !== 0) { - console.warn('exit code:', code); - } - }; - this._exitPromise = new Promise((resolve) => { - this._resolveExitPromise = resolve; - }); - this._pendingEvent = null; - this._scheduledTimeouts = new Map(); - this._nextCallbackTimeoutID = 1; - - const setInt64 = (addr, v) => { - this.mem.setUint32(addr + 0, v, true); - this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true); - }; - - const getInt64 = (addr) => { - const low = this.mem.getUint32(addr + 0, true); - const high = this.mem.getInt32(addr + 4, true); - return low + high * 4294967296; - }; - - const loadValue = (addr) => { - const f = this.mem.getFloat64(addr, true); - if (f === 0) { - return undefined; - } - if (!isNaN(f)) { - return f; - } - - const id = this.mem.getUint32(addr, true); - return this._values[id]; - }; - - const storeValue = (addr, v) => { - const nanHead = 0x7ff80000; - - if (typeof v === 'number' && v !== 0) { - if (isNaN(v)) { - this.mem.setUint32(addr + 4, nanHead, true); - this.mem.setUint32(addr, 0, true); - return; - } - this.mem.setFloat64(addr, v, true); - return; - } - - if (v === undefined) { - this.mem.setFloat64(addr, 0, true); - return; - } - - let id = this._ids.get(v); - if (id === undefined) { - id = this._idPool.pop(); - if (id === undefined) { - id = this._values.length; - } - this._values[id] = v; - this._goRefCounts[id] = 0; - this._ids.set(v, id); - } - this._goRefCounts[id]++; - let typeFlag = 0; - switch (typeof v) { - case 'object': - if (v !== null) { - typeFlag = 1; - } - break; - case 'string': - typeFlag = 2; - break; - case 'symbol': - typeFlag = 3; - break; - case 'function': - typeFlag = 4; - break; - } - this.mem.setUint32(addr + 4, nanHead | typeFlag, true); - this.mem.setUint32(addr, id, true); - }; - - const loadSlice = (addr) => { - const array = getInt64(addr + 0); - const len = getInt64(addr + 8); - return new Uint8Array(this._inst.exports.mem.buffer, array, len); - }; - - const loadSliceOfValues = (addr) => { - const array = getInt64(addr + 0); - const len = getInt64(addr + 8); - const a = new Array(len); - for (let i = 0; i < len; i++) { - a[i] = loadValue(array + i * 8); - } - return a; - }; - - const loadString = (addr) => { - const saddr = getInt64(addr + 0); - const len = getInt64(addr + 8); - return decoder.decode( - new DataView(this._inst.exports.mem.buffer, saddr, len) - ); - }; - - const timeOrigin = Date.now() - performance.now(); - this.importObject = { - go: { - // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters) - // may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported - // function. A goroutine can switch to a new stack if the current stack is too small (see morestack function). - // This changes the SP, thus we have to update the SP used by the imported function. - - // func wasmExit(code int32) - 'runtime.wasmExit': (sp) => { - sp >>>= 0; - const code = this.mem.getInt32(sp + 8, true); - this.exited = true; - delete this._inst; - delete this._values; - delete this._goRefCounts; - delete this._ids; - delete this._idPool; - this.exit(code); - }, - - // func wasmWrite(fd uintptr, p unsafe.Pointer, n int32) - 'runtime.wasmWrite': (sp) => { - sp >>>= 0; - const fd = getInt64(sp + 8); - const p = getInt64(sp + 16); - const n = this.mem.getInt32(sp + 24, true); - fs.writeSync( - fd, - new Uint8Array(this._inst.exports.mem.buffer, p, n) - ); - }, - - // func resetMemoryDataView() - 'runtime.resetMemoryDataView': (sp) => { - sp >>>= 0; - this.mem = new DataView(this._inst.exports.mem.buffer); - }, - - // func nanotime1() int64 - 'runtime.nanotime1': (sp) => { - sp >>>= 0; - setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000); - }, - - // func walltime() (sec int64, nsec int32) - 'runtime.walltime': (sp) => { - sp >>>= 0; - const msec = new Date().getTime(); - setInt64(sp + 8, msec / 1000); - this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true); - }, - - // func scheduleTimeoutEvent(delay int64) int32 - 'runtime.scheduleTimeoutEvent': (sp) => { - sp >>>= 0; - const id = this._nextCallbackTimeoutID; - this._nextCallbackTimeoutID++; - this._scheduledTimeouts.set( - id, - setTimeout( - () => { - this._resume(); - while (this._scheduledTimeouts.has(id)) { - // for some reason Go failed to register the timeout event, log and try again - // (temporary workaround for https://github.com/golang/go/issues/28975) - console.warn('scheduleTimeoutEvent: missed timeout event'); - this._resume(); - } - }, - getInt64(sp + 8) + 1 // setTimeout has been seen to fire up to 1 millisecond early - ) - ); - this.mem.setInt32(sp + 16, id, true); - }, - - // func clearTimeoutEvent(id int32) - 'runtime.clearTimeoutEvent': (sp) => { - sp >>>= 0; - const id = this.mem.getInt32(sp + 8, true); - clearTimeout(this._scheduledTimeouts.get(id)); - this._scheduledTimeouts.delete(id); - }, - - // func getRandomData(r []byte) - 'runtime.getRandomData': (sp) => { - sp >>>= 0; - crypto.getRandomValues(loadSlice(sp + 8)); - }, - - // func finalizeRef(v ref) - 'syscall/js.finalizeRef': (sp) => { - sp >>>= 0; - const id = this.mem.getUint32(sp + 8, true); - this._goRefCounts[id]--; - if (this._goRefCounts[id] === 0) { - const v = this._values[id]; - this._values[id] = null; - this._ids.delete(v); - this._idPool.push(id); - } - }, - - // func stringVal(value string) ref - 'syscall/js.stringVal': (sp) => { - sp >>>= 0; - storeValue(sp + 24, loadString(sp + 8)); - }, - - // func valueGet(v ref, p string) ref - 'syscall/js.valueGet': (sp) => { - sp >>>= 0; - const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16)); - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 32, result); - }, - - // func valueSet(v ref, p string, x ref) - 'syscall/js.valueSet': (sp) => { - sp >>>= 0; - Reflect.set( - loadValue(sp + 8), - loadString(sp + 16), - loadValue(sp + 32) - ); - }, - - // func valueDelete(v ref, p string) - 'syscall/js.valueDelete': (sp) => { - sp >>>= 0; - Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16)); - }, - - // func valueIndex(v ref, i int) ref - 'syscall/js.valueIndex': (sp) => { - sp >>>= 0; - storeValue( - sp + 24, - Reflect.get(loadValue(sp + 8), getInt64(sp + 16)) - ); - }, - - // valueSetIndex(v ref, i int, x ref) - 'syscall/js.valueSetIndex': (sp) => { - sp >>>= 0; - Reflect.set( - loadValue(sp + 8), - getInt64(sp + 16), - loadValue(sp + 24) - ); - }, - - // func valueCall(v ref, m string, args []ref) (ref, bool) - 'syscall/js.valueCall': (sp) => { - sp >>>= 0; - try { - const v = loadValue(sp + 8); - const m = Reflect.get(v, loadString(sp + 16)); - const args = loadSliceOfValues(sp + 32); - const result = Reflect.apply(m, v, args); - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 56, result); - this.mem.setUint8(sp + 64, 1); - } catch (err) { - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 56, err); - this.mem.setUint8(sp + 64, 0); - } - }, - - // func valueInvoke(v ref, args []ref) (ref, bool) - 'syscall/js.valueInvoke': (sp) => { - sp >>>= 0; - try { - const v = loadValue(sp + 8); - const args = loadSliceOfValues(sp + 16); - const result = Reflect.apply(v, undefined, args); - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 40, result); - this.mem.setUint8(sp + 48, 1); - } catch (err) { - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 40, err); - this.mem.setUint8(sp + 48, 0); - } - }, - - // func valueNew(v ref, args []ref) (ref, bool) - 'syscall/js.valueNew': (sp) => { - sp >>>= 0; - try { - const v = loadValue(sp + 8); - const args = loadSliceOfValues(sp + 16); - const result = Reflect.construct(v, args); - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 40, result); - this.mem.setUint8(sp + 48, 1); - } catch (err) { - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 40, err); - this.mem.setUint8(sp + 48, 0); - } - }, - - // func valueLength(v ref) int - 'syscall/js.valueLength': (sp) => { - sp >>>= 0; - setInt64(sp + 16, parseInt(loadValue(sp + 8).length)); - }, - - // valuePrepareString(v ref) (ref, int) - 'syscall/js.valuePrepareString': (sp) => { - sp >>>= 0; - const str = encoder.encode(String(loadValue(sp + 8))); - storeValue(sp + 16, str); - setInt64(sp + 24, str.length); - }, - - // valueLoadString(v ref, b []byte) - 'syscall/js.valueLoadString': (sp) => { - sp >>>= 0; - const str = loadValue(sp + 8); - loadSlice(sp + 16).set(str); - }, - - // func valueInstanceOf(v ref, t ref) bool - 'syscall/js.valueInstanceOf': (sp) => { - sp >>>= 0; - this.mem.setUint8( - sp + 24, - loadValue(sp + 8) instanceof loadValue(sp + 16) ? 1 : 0 - ); - }, - - // func copyBytesToGo(dst []byte, src ref) (int, bool) - 'syscall/js.copyBytesToGo': (sp) => { - sp >>>= 0; - const dst = loadSlice(sp + 8); - const src = loadValue(sp + 32); - if ( - !(src instanceof Uint8Array || src instanceof Uint8ClampedArray) - ) { - this.mem.setUint8(sp + 48, 0); - return; - } - const toCopy = src.subarray(0, dst.length); - dst.set(toCopy); - setInt64(sp + 40, toCopy.length); - this.mem.setUint8(sp + 48, 1); - }, - - // func copyBytesToJS(dst ref, src []byte) (int, bool) - 'syscall/js.copyBytesToJS': (sp) => { - sp >>>= 0; - const dst = loadValue(sp + 8); - const src = loadSlice(sp + 16); - if ( - !(dst instanceof Uint8Array || dst instanceof Uint8ClampedArray) - ) { - this.mem.setUint8(sp + 48, 0); - return; - } - const toCopy = src.subarray(0, dst.length); - dst.set(toCopy); - setInt64(sp + 40, toCopy.length); - this.mem.setUint8(sp + 48, 1); - }, - - debug: (value) => { - console.log(value); - }, - }, - }; - } - - async run(instance) { - if (!(instance instanceof WebAssembly.Instance)) { - throw new Error('Go.run: WebAssembly.Instance expected'); - } - this._inst = instance; - this.mem = new DataView(this._inst.exports.mem.buffer); - this._values = [ - // JS values that Go currently has references to, indexed by reference id - NaN, - 0, - null, - true, - false, - globalThis, - this, - ]; - this._goRefCounts = new Array(this._values.length).fill(Infinity); // number of references that Go has to a JS value, indexed by reference id - this._ids = new Map([ - // mapping from JS values to reference ids - [0, 1], - [null, 2], - [true, 3], - [false, 4], - [globalThis, 5], - [this, 6], - ]); - this._idPool = []; // unused ids that have been garbage collected - this.exited = false; // whether the Go program has exited - - // Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory. - let offset = 4096; - - const strPtr = (str) => { - const ptr = offset; - const bytes = encoder.encode(str + '\0'); - new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes); - offset += bytes.length; - if (offset % 8 !== 0) { - offset += 8 - (offset % 8); - } - return ptr; - }; - - const argc = this.argv.length; - - const argvPtrs = []; - this.argv.forEach((arg) => { - argvPtrs.push(strPtr(arg)); - }); - argvPtrs.push(0); - - const keys = Object.keys(this.env).sort(); - keys.forEach((key) => { - argvPtrs.push(strPtr(`${key}=${this.env[key]}`)); - }); - argvPtrs.push(0); - - const argv = offset; - argvPtrs.forEach((ptr) => { - this.mem.setUint32(offset, ptr, true); - this.mem.setUint32(offset + 4, 0, true); - offset += 8; - }); - - // The linker guarantees global data starts from at least wasmMinDataAddr. - // Keep in sync with cmd/link/internal/ld/data.go:wasmMinDataAddr. - const wasmMinDataAddr = 4096 + 8192; - if (offset >= wasmMinDataAddr) { - throw new Error( - 'total length of command line and environment variables exceeds limit' - ); - } - - this._inst.exports.run(argc, argv); - if (this.exited) { - this._resolveExitPromise(); - } - await this._exitPromise; - } - - _resume() { - if (this.exited) { - throw new Error('Go program has already exited'); - } - this._inst.exports.resume(); - if (this.exited) { - this._resolveExitPromise(); - } - } - - _makeFuncWrapper(id) { - const go = this; - return function () { - const event = { id: id, this: this, args: arguments }; - go._pendingEvent = event; - go._resume(); - return event.result; - }; - } - }; + const enosys = () => { + const err = new Error("not implemented"); + err.code = "ENOSYS"; + return err; + }; + + if (!globalThis.fs) { + let outputBuf = ""; + globalThis.fs = { + constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused + writeSync(fd, buf) { + outputBuf += decoder.decode(buf); + const nl = outputBuf.lastIndexOf("\n"); + if (nl != -1) { + console.log(outputBuf.substring(0, nl)); + outputBuf = outputBuf.substring(nl + 1); + } + return buf.length; + }, + write(fd, buf, offset, length, position, callback) { + if (offset !== 0 || length !== buf.length || position !== null) { + callback(enosys()); + return; + } + const n = this.writeSync(fd, buf); + callback(null, n); + }, + chmod(path, mode, callback) { callback(enosys()); }, + chown(path, uid, gid, callback) { callback(enosys()); }, + close(fd, callback) { callback(enosys()); }, + fchmod(fd, mode, callback) { callback(enosys()); }, + fchown(fd, uid, gid, callback) { callback(enosys()); }, + fstat(fd, callback) { callback(enosys()); }, + fsync(fd, callback) { callback(null); }, + ftruncate(fd, length, callback) { callback(enosys()); }, + lchown(path, uid, gid, callback) { callback(enosys()); }, + link(path, link, callback) { callback(enosys()); }, + lstat(path, callback) { callback(enosys()); }, + mkdir(path, perm, callback) { callback(enosys()); }, + open(path, flags, mode, callback) { callback(enosys()); }, + read(fd, buffer, offset, length, position, callback) { callback(enosys()); }, + readdir(path, callback) { callback(enosys()); }, + readlink(path, callback) { callback(enosys()); }, + rename(from, to, callback) { callback(enosys()); }, + rmdir(path, callback) { callback(enosys()); }, + stat(path, callback) { callback(enosys()); }, + symlink(path, link, callback) { callback(enosys()); }, + truncate(path, length, callback) { callback(enosys()); }, + unlink(path, callback) { callback(enosys()); }, + utimes(path, atime, mtime, callback) { callback(enosys()); }, + }; + } + + if (!globalThis.process) { + globalThis.process = { + getuid() { return -1; }, + getgid() { return -1; }, + geteuid() { return -1; }, + getegid() { return -1; }, + getgroups() { throw enosys(); }, + pid: -1, + ppid: -1, + umask() { throw enosys(); }, + cwd() { throw enosys(); }, + chdir() { throw enosys(); }, + } + } + + if (!globalThis.crypto) { + throw new Error("globalThis.crypto is not available, polyfill required (crypto.getRandomValues only)"); + } + + if (!globalThis.performance) { + throw new Error("globalThis.performance is not available, polyfill required (performance.now only)"); + } + + if (!globalThis.TextEncoder) { + throw new Error("globalThis.TextEncoder is not available, polyfill required"); + } + + if (!globalThis.TextDecoder) { + throw new Error("globalThis.TextDecoder is not available, polyfill required"); + } + + const encoder = new TextEncoder("utf-8"); + const decoder = new TextDecoder("utf-8"); + + globalThis.Go = class { + constructor() { + this.argv = ["js"]; + this.env = {}; + this.exit = (code) => { + if (code !== 0) { + console.warn("exit code:", code); + } + }; + this._exitPromise = new Promise((resolve) => { + this._resolveExitPromise = resolve; + }); + this._pendingEvent = null; + this._scheduledTimeouts = new Map(); + this._nextCallbackTimeoutID = 1; + + const setInt64 = (addr, v) => { + this.mem.setUint32(addr + 0, v, true); + this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true); + } + + const setInt32 = (addr, v) => { + this.mem.setUint32(addr + 0, v, true); + } + + const getInt64 = (addr) => { + const low = this.mem.getUint32(addr + 0, true); + const high = this.mem.getInt32(addr + 4, true); + return low + high * 4294967296; + } + + const loadValue = (addr) => { + const f = this.mem.getFloat64(addr, true); + if (f === 0) { + return undefined; + } + if (!isNaN(f)) { + return f; + } + + const id = this.mem.getUint32(addr, true); + return this._values[id]; + } + + const storeValue = (addr, v) => { + const nanHead = 0x7FF80000; + + if (typeof v === "number" && v !== 0) { + if (isNaN(v)) { + this.mem.setUint32(addr + 4, nanHead, true); + this.mem.setUint32(addr, 0, true); + return; + } + this.mem.setFloat64(addr, v, true); + return; + } + + if (v === undefined) { + this.mem.setFloat64(addr, 0, true); + return; + } + + let id = this._ids.get(v); + if (id === undefined) { + id = this._idPool.pop(); + if (id === undefined) { + id = this._values.length; + } + this._values[id] = v; + this._goRefCounts[id] = 0; + this._ids.set(v, id); + } + this._goRefCounts[id]++; + let typeFlag = 0; + switch (typeof v) { + case "object": + if (v !== null) { + typeFlag = 1; + } + break; + case "string": + typeFlag = 2; + break; + case "symbol": + typeFlag = 3; + break; + case "function": + typeFlag = 4; + break; + } + this.mem.setUint32(addr + 4, nanHead | typeFlag, true); + this.mem.setUint32(addr, id, true); + } + + const loadSlice = (addr) => { + const array = getInt64(addr + 0); + const len = getInt64(addr + 8); + return new Uint8Array(this._inst.exports.mem.buffer, array, len); + } + + const loadSliceOfValues = (addr) => { + const array = getInt64(addr + 0); + const len = getInt64(addr + 8); + const a = new Array(len); + for (let i = 0; i < len; i++) { + a[i] = loadValue(array + i * 8); + } + return a; + } + + const loadString = (addr) => { + const saddr = getInt64(addr + 0); + const len = getInt64(addr + 8); + return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len)); + } + + const timeOrigin = Date.now() - performance.now(); + this.importObject = { + _gotest: { + add: (a, b) => a + b, + }, + gojs: { + // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters) + // may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported + // function. A goroutine can switch to a new stack if the current stack is too small (see morestack function). + // This changes the SP, thus we have to update the SP used by the imported function. + + // func wasmExit(code int32) + "runtime.wasmExit": (sp) => { + sp >>>= 0; + const code = this.mem.getInt32(sp + 8, true); + this.exited = true; + delete this._inst; + delete this._values; + delete this._goRefCounts; + delete this._ids; + delete this._idPool; + this.exit(code); + }, + + // func wasmWrite(fd uintptr, p unsafe.Pointer, n int32) + "runtime.wasmWrite": (sp) => { + sp >>>= 0; + const fd = getInt64(sp + 8); + const p = getInt64(sp + 16); + const n = this.mem.getInt32(sp + 24, true); + fs.writeSync(fd, new Uint8Array(this._inst.exports.mem.buffer, p, n)); + }, + + // func resetMemoryDataView() + "runtime.resetMemoryDataView": (sp) => { + sp >>>= 0; + this.mem = new DataView(this._inst.exports.mem.buffer); + }, + + // func nanotime1() int64 + "runtime.nanotime1": (sp) => { + sp >>>= 0; + setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000); + }, + + // func walltime() (sec int64, nsec int32) + "runtime.walltime": (sp) => { + sp >>>= 0; + const msec = (new Date).getTime(); + setInt64(sp + 8, msec / 1000); + this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true); + }, + + // func scheduleTimeoutEvent(delay int64) int32 + "runtime.scheduleTimeoutEvent": (sp) => { + sp >>>= 0; + const id = this._nextCallbackTimeoutID; + this._nextCallbackTimeoutID++; + this._scheduledTimeouts.set(id, setTimeout( + () => { + this._resume(); + while (this._scheduledTimeouts.has(id)) { + // for some reason Go failed to register the timeout event, log and try again + // (temporary workaround for https://github.com/golang/go/issues/28975) + console.warn("scheduleTimeoutEvent: missed timeout event"); + this._resume(); + } + }, + getInt64(sp + 8), + )); + this.mem.setInt32(sp + 16, id, true); + }, + + // func clearTimeoutEvent(id int32) + "runtime.clearTimeoutEvent": (sp) => { + sp >>>= 0; + const id = this.mem.getInt32(sp + 8, true); + clearTimeout(this._scheduledTimeouts.get(id)); + this._scheduledTimeouts.delete(id); + }, + + // func getRandomData(r []byte) + "runtime.getRandomData": (sp) => { + sp >>>= 0; + crypto.getRandomValues(loadSlice(sp + 8)); + }, + + // func finalizeRef(v ref) + "syscall/js.finalizeRef": (sp) => { + sp >>>= 0; + const id = this.mem.getUint32(sp + 8, true); + this._goRefCounts[id]--; + if (this._goRefCounts[id] === 0) { + const v = this._values[id]; + this._values[id] = null; + this._ids.delete(v); + this._idPool.push(id); + } + }, + + // func stringVal(value string) ref + "syscall/js.stringVal": (sp) => { + sp >>>= 0; + storeValue(sp + 24, loadString(sp + 8)); + }, + + // func valueGet(v ref, p string) ref + "syscall/js.valueGet": (sp) => { + sp >>>= 0; + const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16)); + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 32, result); + }, + + // func valueSet(v ref, p string, x ref) + "syscall/js.valueSet": (sp) => { + sp >>>= 0; + Reflect.set(loadValue(sp + 8), loadString(sp + 16), loadValue(sp + 32)); + }, + + // func valueDelete(v ref, p string) + "syscall/js.valueDelete": (sp) => { + sp >>>= 0; + Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16)); + }, + + // func valueIndex(v ref, i int) ref + "syscall/js.valueIndex": (sp) => { + sp >>>= 0; + storeValue(sp + 24, Reflect.get(loadValue(sp + 8), getInt64(sp + 16))); + }, + + // valueSetIndex(v ref, i int, x ref) + "syscall/js.valueSetIndex": (sp) => { + sp >>>= 0; + Reflect.set(loadValue(sp + 8), getInt64(sp + 16), loadValue(sp + 24)); + }, + + // func valueCall(v ref, m string, args []ref) (ref, bool) + "syscall/js.valueCall": (sp) => { + sp >>>= 0; + try { + const v = loadValue(sp + 8); + const m = Reflect.get(v, loadString(sp + 16)); + const args = loadSliceOfValues(sp + 32); + const result = Reflect.apply(m, v, args); + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 56, result); + this.mem.setUint8(sp + 64, 1); + } catch (err) { + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 56, err); + this.mem.setUint8(sp + 64, 0); + } + }, + + // func valueInvoke(v ref, args []ref) (ref, bool) + "syscall/js.valueInvoke": (sp) => { + sp >>>= 0; + try { + const v = loadValue(sp + 8); + const args = loadSliceOfValues(sp + 16); + const result = Reflect.apply(v, undefined, args); + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 40, result); + this.mem.setUint8(sp + 48, 1); + } catch (err) { + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 40, err); + this.mem.setUint8(sp + 48, 0); + } + }, + + // func valueNew(v ref, args []ref) (ref, bool) + "syscall/js.valueNew": (sp) => { + sp >>>= 0; + try { + const v = loadValue(sp + 8); + const args = loadSliceOfValues(sp + 16); + const result = Reflect.construct(v, args); + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 40, result); + this.mem.setUint8(sp + 48, 1); + } catch (err) { + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 40, err); + this.mem.setUint8(sp + 48, 0); + } + }, + + // func valueLength(v ref) int + "syscall/js.valueLength": (sp) => { + sp >>>= 0; + setInt64(sp + 16, parseInt(loadValue(sp + 8).length)); + }, + + // valuePrepareString(v ref) (ref, int) + "syscall/js.valuePrepareString": (sp) => { + sp >>>= 0; + const str = encoder.encode(String(loadValue(sp + 8))); + storeValue(sp + 16, str); + setInt64(sp + 24, str.length); + }, + + // valueLoadString(v ref, b []byte) + "syscall/js.valueLoadString": (sp) => { + sp >>>= 0; + const str = loadValue(sp + 8); + loadSlice(sp + 16).set(str); + }, + + // func valueInstanceOf(v ref, t ref) bool + "syscall/js.valueInstanceOf": (sp) => { + sp >>>= 0; + this.mem.setUint8(sp + 24, (loadValue(sp + 8) instanceof loadValue(sp + 16)) ? 1 : 0); + }, + + // func copyBytesToGo(dst []byte, src ref) (int, bool) + "syscall/js.copyBytesToGo": (sp) => { + sp >>>= 0; + const dst = loadSlice(sp + 8); + const src = loadValue(sp + 32); + if (!(src instanceof Uint8Array || src instanceof Uint8ClampedArray)) { + this.mem.setUint8(sp + 48, 0); + return; + } + const toCopy = src.subarray(0, dst.length); + dst.set(toCopy); + setInt64(sp + 40, toCopy.length); + this.mem.setUint8(sp + 48, 1); + }, + + // func copyBytesToJS(dst ref, src []byte) (int, bool) + "syscall/js.copyBytesToJS": (sp) => { + sp >>>= 0; + const dst = loadValue(sp + 8); + const src = loadSlice(sp + 16); + if (!(dst instanceof Uint8Array || dst instanceof Uint8ClampedArray)) { + this.mem.setUint8(sp + 48, 0); + return; + } + const toCopy = src.subarray(0, dst.length); + dst.set(toCopy); + setInt64(sp + 40, toCopy.length); + this.mem.setUint8(sp + 48, 1); + }, + + "debug": (value) => { + console.log(value); + }, + } + }; + } + + async run(instance) { + if (!(instance instanceof WebAssembly.Instance)) { + throw new Error("Go.run: WebAssembly.Instance expected"); + } + this._inst = instance; + this.mem = new DataView(this._inst.exports.mem.buffer); + this._values = [ // JS values that Go currently has references to, indexed by reference id + NaN, + 0, + null, + true, + false, + globalThis, + this, + ]; + this._goRefCounts = new Array(this._values.length).fill(Infinity); // number of references that Go has to a JS value, indexed by reference id + this._ids = new Map([ // mapping from JS values to reference ids + [0, 1], + [null, 2], + [true, 3], + [false, 4], + [globalThis, 5], + [this, 6], + ]); + this._idPool = []; // unused ids that have been garbage collected + this.exited = false; // whether the Go program has exited + + // Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory. + let offset = 4096; + + const strPtr = (str) => { + const ptr = offset; + const bytes = encoder.encode(str + "\0"); + new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes); + offset += bytes.length; + if (offset % 8 !== 0) { + offset += 8 - (offset % 8); + } + return ptr; + }; + + const argc = this.argv.length; + + const argvPtrs = []; + this.argv.forEach((arg) => { + argvPtrs.push(strPtr(arg)); + }); + argvPtrs.push(0); + + const keys = Object.keys(this.env).sort(); + keys.forEach((key) => { + argvPtrs.push(strPtr(`${key}=${this.env[key]}`)); + }); + argvPtrs.push(0); + + const argv = offset; + argvPtrs.forEach((ptr) => { + this.mem.setUint32(offset, ptr, true); + this.mem.setUint32(offset + 4, 0, true); + offset += 8; + }); + + // The linker guarantees global data starts from at least wasmMinDataAddr. + // Keep in sync with cmd/link/internal/ld/data.go:wasmMinDataAddr. + const wasmMinDataAddr = 4096 + 8192; + if (offset >= wasmMinDataAddr) { + throw new Error("total length of command line and environment variables exceeds limit"); + } + + this._inst.exports.run(argc, argv); + if (this.exited) { + this._resolveExitPromise(); + } + await this._exitPromise; + } + + _resume() { + if (this.exited) { + throw new Error("Go program has already exited"); + } + this._inst.exports.resume(); + if (this.exited) { + this._resolveExitPromise(); + } + } + + _makeFuncWrapper(id) { + const go = this; + return function () { + const event = { id: id, this: this, args: arguments }; + go._pendingEvent = event; + go._resume(); + return event.result; + }; + } + } })(); diff --git a/packages/zed-wasm/lib/wasm_exec_node.js b/packages/zed-wasm/lib/wasm_exec_node.js index bf0b5cab3c..986069087b 100644 --- a/packages/zed-wasm/lib/wasm_exec_node.js +++ b/packages/zed-wasm/lib/wasm_exec_node.js @@ -2,51 +2,38 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -'use strict'; +"use strict"; if (process.argv.length < 3) { - console.error('usage: go_js_wasm_exec [wasm binary] [arguments]'); - process.exit(1); + console.error("usage: go_js_wasm_exec [wasm binary] [arguments]"); + process.exit(1); } globalThis.require = require; -globalThis.fs = require('fs'); -globalThis.TextEncoder = require('util').TextEncoder; -globalThis.TextDecoder = require('util').TextDecoder; +globalThis.fs = require("fs"); +globalThis.TextEncoder = require("util").TextEncoder; +globalThis.TextDecoder = require("util").TextDecoder; -globalThis.performance = { - now() { - const [sec, nsec] = process.hrtime(); - return sec * 1000 + nsec / 1000000; - }, -}; +globalThis.performance ??= require("performance"); -const crypto = require('crypto'); -globalThis.crypto = { - getRandomValues(b) { - crypto.randomFillSync(b); - }, -}; +globalThis.crypto ??= require("crypto"); -require('./wasm_exec'); +require("./wasm_exec"); const go = new Go(); go.argv = process.argv.slice(2); -go.env = Object.assign({ TMPDIR: require('os').tmpdir() }, process.env); +go.env = Object.assign({ TMPDIR: require("os").tmpdir() }, process.env); go.exit = process.exit; -WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject) - .then((result) => { - process.on('exit', (code) => { - // Node.js exits if no event handler is pending - if (code === 0 && !go.exited) { - // deadlock, make Go print error and stack traces - go._pendingEvent = { id: 0 }; - go._resume(); - } - }); - return go.run(result.instance); - }) - .catch((err) => { - console.error(err); - process.exit(1); - }); +WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then((result) => { + process.on("exit", (code) => { // Node.js exits if no event handler is pending + if (code === 0 && !go.exited) { + // deadlock, make Go print error and stack traces + go._pendingEvent = { id: 0 }; + go._resume(); + } + }); + return go.run(result.instance); +}).catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/packages/zed-wasm/main.go b/packages/zed-wasm/main.go index e0c5997a75..96e05739ef 100644 --- a/packages/zed-wasm/main.go +++ b/packages/zed-wasm/main.go @@ -11,6 +11,7 @@ import ( "github.com/brimdata/zed" "github.com/brimdata/zed/compiler" + "github.com/brimdata/zed/compiler/parser" "github.com/brimdata/zed/pkg/storage" "github.com/brimdata/zed/runtime" "github.com/brimdata/zed/zbuf" @@ -21,6 +22,7 @@ import ( func main() { wasm.Expose("zq", zq) + wasm.Expose("parse", parse) wasm.Ready() <-make(chan struct{}) } @@ -59,7 +61,7 @@ func zq(opts opts) wasm.Promise { return "", errInvalidInput } zctx := zed.NewContext() - zr, err := anyio.NewReaderWithOpts(zctx, r, anyio.ReaderOpts{ + zr, err := anyio.NewReaderWithOpts(zctx, r, nil, anyio.ReaderOpts{ Format: opts.InputFormat, }) if err != nil { @@ -89,6 +91,24 @@ func zq(opts opts) wasm.Promise { }) } +func parse(program string) (interface{}, error) { + ast, err := parser.ParseZed(nil, program) + result := ParseResult{AST: ast} + if err != nil { + var ok bool + result.Error, ok = err.(*parser.Error) + if !ok { + return nil, err + } + } + return result, nil +} + +type ParseResult struct { + AST interface{} `wasm:"ast"` + Error *parser.Error `wasm:"error"` +} + func readableStream(readable js.Value) io.Reader { pr, pw := io.Pipe() go func() { diff --git a/packages/zed-wasm/package.json b/packages/zed-wasm/package.json index dc7e6c405d..52a625132d 100644 --- a/packages/zed-wasm/package.json +++ b/packages/zed-wasm/package.json @@ -2,7 +2,11 @@ "name": "@brimdata/zed-wasm", "version": "0.0.17", "homepage": "https://github.com/brimdata/zed", - "type": "module", + "type": "commonjs", + "exports": "./src/index.js", + "browser": { + "./src/index.js": "./src/browser.js" + }, "keywords": [ "zed", "data", diff --git a/packages/zed-wasm/project.json b/packages/zed-wasm/project.json index 0535ae092a..886af42373 100644 --- a/packages/zed-wasm/project.json +++ b/packages/zed-wasm/project.json @@ -5,20 +5,42 @@ "projectType": "library", "targets": { "build": { + "executor": "nx:run-commands", + "options": { + "outputPath": "dist/packages/zed-js", + "commands": [] + }, + "dependsOn": ["build-node", "build-browser"] + }, + "build-browser": { "dependsOn": ["build-go"], "executor": "@nrwl/esbuild:esbuild", "options": { - "main": "packages/zed-wasm/src/index.ts", + "main": "packages/zed-wasm/src/browser.ts", "tsConfig": "packages/zed-wasm/tsconfig.lib.json", "outputPath": "dist/packages/zed-wasm", "assets": ["packages/zed-wasm/*.md"], "format": ["esm"], "deleteOutputPath": false, + "thirdParty": true, "esbuildOptions": { "inject": ["packages/zed-wasm/lib/global-shim.js"] } } }, + "build-node": { + "dependsOn": ["build-go"], + "executor": "@nrwl/js:tsc", + "outputs": ["{options.outputPath}"], + "options": { + "main": "packages/zed-wasm/src/index.ts", + "tsConfig": "packages/zed-wasm/tsconfig.lib.json", + "outputPath": "dist/packages/zed-wasm", + "assets": ["packages/zed-wasm/*.md"], + "buildableProjectDepsInPackageJsonType": "dependencies", + "clean": false + } + }, "build-go": { "executor": "nx:run-commands", "outputs": ["dist/packages/zed-wasm/main.wasm"], diff --git a/packages/zed-wasm/src/browser.ts b/packages/zed-wasm/src/browser.ts new file mode 100644 index 0000000000..c63326a697 --- /dev/null +++ b/packages/zed-wasm/src/browser.ts @@ -0,0 +1,39 @@ +import '../lib/wasm_exec'; +import bridge from 'golang-wasm/src/bridge'; +import { LoadFormat, decode, ndjson } from '@brimdata/zed-js'; +import { getInput } from './input'; + +let go: any; + +async function loadWasm() { + const url = new URL('.', import.meta.url); + const path = url.href + 'main.wasm'; + const wasm = await fetch(path); + console.log('wasm', wasm); + go = bridge(wasm.arrayBuffer()); +} + +export async function zq(opts: { + program?: string; + input?: string | File | Blob | ReadableStream | any[]; + inputFormat?: LoadFormat; + outputFormat?: 'js' | 'zed'; +}) { + if (!go) await loadWasm(); + const result = await go.zq({ + input: getInput(opts.input), + inputFormat: opts.inputFormat, + program: opts.program, + outputFormat: 'zjson', + }); + + const zed = decode(ndjson.parseLines(result)); + if (opts.outputFormat === 'zed') return zed; + return zed.map((val) => val.toJS()); +} + +export async function parse(query: string) { + if (!go) await loadWasm(); + return go.parse(query); +} + diff --git a/packages/zed-wasm/src/index.ts b/packages/zed-wasm/src/index.ts index ca4cbcb5ac..26d0b4f0d6 100644 --- a/packages/zed-wasm/src/index.ts +++ b/packages/zed-wasm/src/index.ts @@ -1,11 +1,14 @@ -import '../lib/wasm_exec'; +import '../lib/wasm_exec_node'; import bridge from 'golang-wasm/src/bridge'; import { LoadFormat, decode, ndjson } from '@brimdata/zed-js'; +import { getInput } from './input'; +import { readFile } from 'fs/promises'; -const url = new URL('.', import.meta.url); -const path = url.href + 'main.wasm'; -const wasm = await fetch(path); -const go = bridge(wasm.arrayBuffer()); +let go: any; + +async function loadWasm() { + go = bridge(readFile('./main.wasm')); +} export async function zq(opts: { program?: string; @@ -13,6 +16,7 @@ export async function zq(opts: { inputFormat?: LoadFormat; outputFormat?: 'js' | 'zed'; }) { + if (!go) await loadWasm(); const result = await go.zq({ input: getInput(opts.input), inputFormat: opts.inputFormat, @@ -25,28 +29,7 @@ export async function zq(opts: { return zed.map((val) => val.toJS()); } -function getInput( - input: string | File | Blob | ReadableStream | Response | undefined | any[] -) { - if (typeof input === 'string') return input; - if (input instanceof File) return input.stream(); - if (input instanceof Blob) return input.stream(); - if (input instanceof ReadableStream) return input; - if (input instanceof Response) return input.body; - if (Array.isArray(input)) return arrayStream(input); - if (input === undefined) return undefined; - if (input === null) return undefined; - throw new Error(`Unsupported input type provided to zq ${input}`); -} - -function arrayStream(input: any[]) { - const encoder = new TextEncoder(); - return new ReadableStream({ - start(ctl) { - for (const item of input) { - ctl.enqueue(encoder.encode(JSON.stringify(item) + '\n')); - } - ctl.close(); - }, - }); +export async function parse(query: string) { + if (!go) await loadWasm(); + return go.parse(query); } diff --git a/packages/zed-wasm/src/input.ts b/packages/zed-wasm/src/input.ts new file mode 100644 index 0000000000..0d3880f567 --- /dev/null +++ b/packages/zed-wasm/src/input.ts @@ -0,0 +1,25 @@ +export function getInput( + input: string | File | Blob | ReadableStream | Response | undefined | any[] +) { + if (typeof input === 'string') return input; + if (input instanceof File) return input.stream(); + if (input instanceof Blob) return input.stream(); + if (input instanceof ReadableStream) return input; + if (input instanceof Response) return input.body; + if (Array.isArray(input)) return arrayStream(input); + if (input === undefined) return undefined; + if (input === null) return undefined; + throw new Error(`Unsupported input type provided to zq ${input}`); +} + +function arrayStream(input: any[]) { + const encoder = new TextEncoder(); + return new ReadableStream({ + start(ctl) { + for (const item of input) { + ctl.enqueue(encoder.encode(JSON.stringify(item) + '\n')); + } + ctl.close(); + }, + }); +} diff --git a/packages/zed-wasm/test/parse.test.js b/packages/zed-wasm/test/parse.test.js new file mode 100644 index 0000000000..8bffa41ce6 --- /dev/null +++ b/packages/zed-wasm/test/parse.test.js @@ -0,0 +1,13 @@ +const assert = chai.assert; + +describe('parse', () => { + it('should parse some shit', async () => { + try { + const resp = await parse('me :='); + console.log('resp', resp) + } + catch (err) { + console.log("error.caught", err) + } + }); +}); diff --git a/packages/zed-wasm/tsconfig.json b/packages/zed-wasm/tsconfig.json index 952b834959..b9a3062984 100644 --- a/packages/zed-wasm/tsconfig.json +++ b/packages/zed-wasm/tsconfig.json @@ -1,14 +1,15 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "target": "es2017", - "module": "es2022", + "module": "ES2022", + "moduleResolution": "node", + "target": "ES2017", "forceConsistentCasingInFileNames": true, "strict": true, "noImplicitOverride": true, "noPropertyAccessFromIndexSignature": true, "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true, + "noFallthroughCasesInSwitch": true }, "files": [], "include": [], diff --git a/packages/zed-wasm/tsconfig.lib.json b/packages/zed-wasm/tsconfig.lib.json index 58b793553b..33eca2c2cd 100644 --- a/packages/zed-wasm/tsconfig.lib.json +++ b/packages/zed-wasm/tsconfig.lib.json @@ -3,7 +3,7 @@ "compilerOptions": { "outDir": "../../dist/out-tsc", "declaration": true, - "lib": ["es2020", "DOM"], + "types": ["node"] }, "include": ["src/**/*.ts"], "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] diff --git a/yarn.lock b/yarn.lock index 6855fd01e4..cd5d087a75 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1930,7 +1930,7 @@ __metadata: languageName: unknown linkType: soft -"@brimdata/zed-wasm@workspace:packages/zed-wasm": +"@brimdata/zed-wasm@workspace:*, @brimdata/zed-wasm@workspace:packages/zed-wasm": version: 0.0.0-use.local resolution: "@brimdata/zed-wasm@workspace:packages/zed-wasm" dependencies: @@ -19161,6 +19161,7 @@ __metadata: "@babel/core": ^7.17.9 "@brimdata/zed-js": "workspace:*" "@brimdata/zed-node": "workspace:*" + "@brimdata/zed-wasm": "workspace:*" "@js-joda/core": ^3.2.0 "@monaco-editor/react": ^4.5.1 "@reduxjs/toolkit": ^1.9.3 From 7e8287791d848d5bf054bd81f36300888138318b Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 11 Jan 2024 11:54:56 -0800 Subject: [PATCH 02/43] New Folder Setup --- .node-version | 2 +- apps/zui/.node-version | 1 - packages/zed-wasm/.env.build-go | 2 - packages/zed-wasm/.gitignore | 5 + packages/zed-wasm/index.html | 73 ---------- packages/zed-wasm/lib/bridge.d.ts | 3 - packages/zed-wasm/lib/bridge.js | 127 ++++++++++++++++++ .../zed-wasm/{src/input.ts => lib/input.js} | 6 +- packages/zed-wasm/lib/node-globals.js | 5 + packages/zed-wasm/lib/node-wasm-path.js | 14 ++ packages/zed-wasm/lib/wasm_exec_node.js | 53 +++----- packages/zed-wasm/package.json | 39 +++++- packages/zed-wasm/playwright.config.ts | 83 ++++++++++++ packages/zed-wasm/project.json | 29 ---- packages/zed-wasm/src/browser.js | 33 +++++ packages/zed-wasm/src/browser.ts | 39 ------ packages/zed-wasm/src/golang-wasm.d.ts | 3 - packages/zed-wasm/src/index.js | 38 ++++++ packages/zed-wasm/src/index.ts | 35 ----- packages/zed-wasm/test/browser/parse.spec.ts | 15 +++ packages/zed-wasm/test/browser/zq.test.js | 16 +++ packages/zed-wasm/test/cjs/parse.test.js | 37 +++++ packages/zed-wasm/test/node/parser.test.js | 29 ++++ packages/zed-wasm/test/node/zq.test.js | 9 ++ packages/zed-wasm/test/pages/parse.html | 50 +++++++ packages/zed-wasm/test/pages/zq.html | 55 ++++++++ packages/zed-wasm/test/parse.test.js | 13 -- packages/zed-wasm/test/zq.test.js | 51 ------- packages/zed-wasm/tsconfig.json | 24 ---- packages/zed-wasm/tsconfig.lib.json | 10 -- yarn.lock | 100 +++++++++++++- 31 files changed, 669 insertions(+), 330 deletions(-) delete mode 100644 apps/zui/.node-version delete mode 100644 packages/zed-wasm/.env.build-go create mode 100644 packages/zed-wasm/.gitignore delete mode 100644 packages/zed-wasm/index.html delete mode 100644 packages/zed-wasm/lib/bridge.d.ts create mode 100644 packages/zed-wasm/lib/bridge.js rename packages/zed-wasm/{src/input.ts => lib/input.js} (82%) create mode 100644 packages/zed-wasm/lib/node-globals.js create mode 100644 packages/zed-wasm/lib/node-wasm-path.js create mode 100644 packages/zed-wasm/playwright.config.ts create mode 100644 packages/zed-wasm/src/browser.js delete mode 100644 packages/zed-wasm/src/browser.ts delete mode 100644 packages/zed-wasm/src/golang-wasm.d.ts create mode 100644 packages/zed-wasm/src/index.js delete mode 100644 packages/zed-wasm/src/index.ts create mode 100644 packages/zed-wasm/test/browser/parse.spec.ts create mode 100644 packages/zed-wasm/test/browser/zq.test.js create mode 100644 packages/zed-wasm/test/cjs/parse.test.js create mode 100644 packages/zed-wasm/test/node/parser.test.js create mode 100644 packages/zed-wasm/test/node/zq.test.js create mode 100644 packages/zed-wasm/test/pages/parse.html create mode 100644 packages/zed-wasm/test/pages/zq.html delete mode 100644 packages/zed-wasm/test/parse.test.js delete mode 100644 packages/zed-wasm/test/zq.test.js delete mode 100644 packages/zed-wasm/tsconfig.json delete mode 100644 packages/zed-wasm/tsconfig.lib.json diff --git a/.node-version b/.node-version index 3027af39c1..87ec8842b1 100644 --- a/.node-version +++ b/.node-version @@ -1 +1 @@ -16.10.0 +18.18.2 diff --git a/apps/zui/.node-version b/apps/zui/.node-version deleted file mode 100644 index 56bfee434b..0000000000 --- a/apps/zui/.node-version +++ /dev/null @@ -1 +0,0 @@ -v16.10.0 diff --git a/packages/zed-wasm/.env.build-go b/packages/zed-wasm/.env.build-go deleted file mode 100644 index 240db5bc31..0000000000 --- a/packages/zed-wasm/.env.build-go +++ /dev/null @@ -1,2 +0,0 @@ -GOARCH=wasm -GOOS=js diff --git a/packages/zed-wasm/.gitignore b/packages/zed-wasm/.gitignore new file mode 100644 index 0000000000..68c5d18f00 --- /dev/null +++ b/packages/zed-wasm/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/packages/zed-wasm/index.html b/packages/zed-wasm/index.html deleted file mode 100644 index a6aa46b708..0000000000 --- a/packages/zed-wasm/index.html +++ /dev/null @@ -1,73 +0,0 @@ - - - - - Mocha Tests - - - - -
-

Manual Testing

-
-

Input File

- -
-
-

Program

- -
-
-

Output Format

- -
-
- -
-
-

View results in the devtools console.

-
-

Unit Tests

-
- - - - - - - - - - - - - - - diff --git a/packages/zed-wasm/lib/bridge.d.ts b/packages/zed-wasm/lib/bridge.d.ts deleted file mode 100644 index b889976ff9..0000000000 --- a/packages/zed-wasm/lib/bridge.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module 'golang-wasm/src/bridge' { - export default function (arg: string): number; -} diff --git a/packages/zed-wasm/lib/bridge.js b/packages/zed-wasm/lib/bridge.js new file mode 100644 index 0000000000..a340abadc7 --- /dev/null +++ b/packages/zed-wasm/lib/bridge.js @@ -0,0 +1,127 @@ +// Initially, the __go_wasm__ object will be an empty object. +const g = globalThis; +if (!g.__go_wasm__) { + g.__go_wasm__ = {}; +} + +/** + * The maximum amount of time that we would expect Wasm to take to initialize. + * If it doesn't initialize after this time, we send a warning to console. + * Most likely something has gone wrong if it takes more than 3 seconds to initialize. + */ +const maxTime = 3 * 1000; + +/** + * bridge is an easier way to refer to the Go WASM object. + */ +const bridge = g.__go_wasm__; + +/** + * Wrapper is used by Go to run all Go functions in JS. + * + * @param {Function} goFunc a function that is expected to return an object of the following specification: + * { + * result: undefined | any // undefined when error is returned, or function returns undefined. + * error: Error | undefined // undefined when no error is present. + * } + * + * @returns {Function} returns a function that take arguments which are used to call the Go function. + */ +function wrapper(goFunc) { + return (...args) => { + const result = goFunc.apply(undefined, args); + if (result.error instanceof Error) { + throw result.error; + } + return result.result; + }; +} + +/** + * Sleep is used when awaiting for Go Wasm to initialize. + * It uses the lowest possible sane delay time (via requestAnimationFrame). + * However, if the window is not focused, requestAnimationFrame never returns. + * A timeout will ensure to be called after 50 ms, regardless of whether or not the tab is in focus. + * + * @returns {Promise} an always-resolving promise when a tick has been completed. + */ +function sleep() { + return new Promise((res) => { + requestAnimationFrame(() => res()); + setTimeout(() => { + res(); + }, 50); + }); +} + +/** + * @param {ArrayBuffer} getBytes a promise that is bytes of the Go Wasm object. + * + * @returns {Proxy} an object that can be used to call WASM's objects and properly parse their results. + * + * All values that want to be retrieved from the proxy, regardless of if they are a function or not, must be retrieved + * as if they are from a function call. + * + * If a non-function value is returned however arguments are provided, a warning will be printed. + */ +export default function (getBytes) { + let proxy; + let go; + + async function init() { + bridge.__wrapper__ = wrapper; + + go = new g.Go(); + let bytes = await getBytes; + let result = await WebAssembly.instantiate(bytes, go.importObject); + go.run(result.instance); + } + + init(); + const id = setTimeout(() => { + if (bridge.__ready__ !== true) { + console.warn( + 'Golang WASM Bridge (__go_wasm__.__ready__) still not true after max time' + ); + } + }, maxTime); + + proxy = new Proxy( + {}, + { + get: (_, key) => { + return (...args) => { + return new Promise(async (res, rej) => { + if (!go || go.exited) { + return rej(new Error('The Go instance is not active.')); + } + while (bridge.__ready__ !== true) { + await sleep(); + } + clearTimeout(id); + + if (typeof bridge[key] !== 'function') { + res(bridge[key]); + + if (args.length !== 0) { + console.warn( + 'Retrieved value from WASM returned function type, however called with arguments.' + ); + } + return; + } + + try { + res(bridge[key].apply(undefined, args)); + } catch (e) { + rej(e); + } + }); + }; + }, + } + ); + + bridge.__proxy__ = proxy; + return proxy; +} diff --git a/packages/zed-wasm/src/input.ts b/packages/zed-wasm/lib/input.js similarity index 82% rename from packages/zed-wasm/src/input.ts rename to packages/zed-wasm/lib/input.js index 0d3880f567..9d62cc2380 100644 --- a/packages/zed-wasm/src/input.ts +++ b/packages/zed-wasm/lib/input.js @@ -1,6 +1,4 @@ -export function getInput( - input: string | File | Blob | ReadableStream | Response | undefined | any[] -) { +export function getInput(input) { if (typeof input === 'string') return input; if (input instanceof File) return input.stream(); if (input instanceof Blob) return input.stream(); @@ -12,7 +10,7 @@ export function getInput( throw new Error(`Unsupported input type provided to zq ${input}`); } -function arrayStream(input: any[]) { +function arrayStream(input) { const encoder = new TextEncoder(); return new ReadableStream({ start(ctl) { diff --git a/packages/zed-wasm/lib/node-globals.js b/packages/zed-wasm/lib/node-globals.js new file mode 100644 index 0000000000..ffe6482b09 --- /dev/null +++ b/packages/zed-wasm/lib/node-globals.js @@ -0,0 +1,5 @@ +import crypto from 'node:crypto'; +import getRandomValues from 'polyfill-crypto.getrandomvalues'; + +globalThis.crypto = { ...crypto, getRandomValues }; +globalThis.requestAnimationFrame = setTimeout; diff --git a/packages/zed-wasm/lib/node-wasm-path.js b/packages/zed-wasm/lib/node-wasm-path.js new file mode 100644 index 0000000000..583ba58c57 --- /dev/null +++ b/packages/zed-wasm/lib/node-wasm-path.js @@ -0,0 +1,14 @@ +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +export function getWasmPath() { + if (import.meta.url) { + // In ESM we go back to the dist + const dir = dirname(fileURLToPath(import.meta.url)); + return join(dir, '../dist/main.wasm'); + } else { + // In CSJ we are already in dist because we got built + const dir = __dirname; + return join(dir, './main.wasm'); + } +} diff --git a/packages/zed-wasm/lib/wasm_exec_node.js b/packages/zed-wasm/lib/wasm_exec_node.js index 986069087b..93d9654ac2 100644 --- a/packages/zed-wasm/lib/wasm_exec_node.js +++ b/packages/zed-wasm/lib/wasm_exec_node.js @@ -1,39 +1,22 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -"use strict"; - -if (process.argv.length < 3) { - console.error("usage: go_js_wasm_exec [wasm binary] [arguments]"); - process.exit(1); -} - -globalThis.require = require; -globalThis.fs = require("fs"); -globalThis.TextEncoder = require("util").TextEncoder; -globalThis.TextDecoder = require("util").TextDecoder; - -globalThis.performance ??= require("performance"); - -globalThis.crypto ??= require("crypto"); - -require("./wasm_exec"); +global.require = require; +global.fs = require('fs'); +global.TextEncoder = require('util').TextEncoder; +global.TextDecoder = require('util').TextDecoder; +global.performance ??= require('performance'); +global.crypto ??= require('crypto'); +const path = require('path'); +require('./wasm_exec'); const go = new Go(); go.argv = process.argv.slice(2); -go.env = Object.assign({ TMPDIR: require("os").tmpdir() }, process.env); +go.env = Object.assign({ TMPDIR: require('os').tmpdir() }, process.env); go.exit = process.exit; -WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then((result) => { - process.on("exit", (code) => { // Node.js exits if no event handler is pending - if (code === 0 && !go.exited) { - // deadlock, make Go print error and stack traces - go._pendingEvent = { id: 0 }; - go._resume(); - } - }); - return go.run(result.instance); -}).catch((err) => { - console.error(err); - process.exit(1); -}); + +const wasmFile = path.join(__dirname, '../dist/main.wasm'); +async function initZedWasm() { + const buffer = global.fs.readFileSync(wasmFile); + const result = await WebAssembly.instantiate(buffer, go.importObject); + return go.run(result.instance); +} + +module.exports = { initZedWasm }; diff --git a/packages/zed-wasm/package.json b/packages/zed-wasm/package.json index 52a625132d..494bfbc535 100644 --- a/packages/zed-wasm/package.json +++ b/packages/zed-wasm/package.json @@ -2,10 +2,30 @@ "name": "@brimdata/zed-wasm", "version": "0.0.17", "homepage": "https://github.com/brimdata/zed", - "type": "commonjs", - "exports": "./src/index.js", - "browser": { - "./src/index.js": "./src/browser.js" + "type": "module", + "files": [ + "dist/browser.js", + "dist/index.js", + "dist/main.wasm", + "src/**/*.js" + ], + "main": "./dist/index.cjs", + "module": "./src/index.js", + "exports": { + "require": "./dist/index.cjs", + "default": "./src/index.js" + }, + "jsdelivr": "./dist/browser.js", + "scripts": { + "test": "run-s 'test:*'", + "test:node": "node --test test/node", + "test:browser": "playwright test", + "test:cjs": "jest test/cjs", + "build": "run-s 'build:*'", + "build:browser": "esbuild src/browser.js --outfile=dist/browser.js --format=esm --bundle", + "build:cjs": "esbuild src/index.js --outfile=dist/index.cjs --format=cjs --bundle --platform=node --packages=external", + "build:go": "GOARCH=wasm GOOS=js go build -tags=noasm -o dist/main.wasm main.go", + "start-test-server": "serve -p 2000" }, "keywords": [ "zed", @@ -15,10 +35,17 @@ "wasm" ], "devDependencies": { + "@playwright/test": "^1.40.1", "@types/golang-wasm": "^1.15.0", - "golang-wasm": "github:teamortix/golang-wasm#master" + "@types/node": "^20.11.0", + "esbuild": "^0.17.12", + "golang-wasm": "github:teamortix/golang-wasm#master", + "jest": "^29.4.1", + "npm-run-all": "^4.1.5", + "serve": "^14.2.1" }, "dependencies": { - "@brimdata/zed-js": "workspace:*" + "@brimdata/zed-js": "../../dist/packages/zed-js", + "polyfill-crypto.getrandomvalues": "^1.0.0" } } diff --git a/packages/zed-wasm/playwright.config.ts b/packages/zed-wasm/playwright.config.ts new file mode 100644 index 0000000000..2236e3d2e7 --- /dev/null +++ b/packages/zed-wasm/playwright.config.ts @@ -0,0 +1,83 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './test/browser', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: 'http://127.0.0.1:2000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + webServer: { + command: 'yarn run start-test-server', + url: 'http://127.0.0.1:2000', + reuseExistingServer: !process.env.CI, + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/packages/zed-wasm/project.json b/packages/zed-wasm/project.json index 886af42373..6d35f9f512 100644 --- a/packages/zed-wasm/project.json +++ b/packages/zed-wasm/project.json @@ -12,35 +12,6 @@ }, "dependsOn": ["build-node", "build-browser"] }, - "build-browser": { - "dependsOn": ["build-go"], - "executor": "@nrwl/esbuild:esbuild", - "options": { - "main": "packages/zed-wasm/src/browser.ts", - "tsConfig": "packages/zed-wasm/tsconfig.lib.json", - "outputPath": "dist/packages/zed-wasm", - "assets": ["packages/zed-wasm/*.md"], - "format": ["esm"], - "deleteOutputPath": false, - "thirdParty": true, - "esbuildOptions": { - "inject": ["packages/zed-wasm/lib/global-shim.js"] - } - } - }, - "build-node": { - "dependsOn": ["build-go"], - "executor": "@nrwl/js:tsc", - "outputs": ["{options.outputPath}"], - "options": { - "main": "packages/zed-wasm/src/index.ts", - "tsConfig": "packages/zed-wasm/tsconfig.lib.json", - "outputPath": "dist/packages/zed-wasm", - "assets": ["packages/zed-wasm/*.md"], - "buildableProjectDepsInPackageJsonType": "dependencies", - "clean": false - } - }, "build-go": { "executor": "nx:run-commands", "outputs": ["dist/packages/zed-wasm/main.wasm"], diff --git a/packages/zed-wasm/src/browser.js b/packages/zed-wasm/src/browser.js new file mode 100644 index 0000000000..d5a516add3 --- /dev/null +++ b/packages/zed-wasm/src/browser.js @@ -0,0 +1,33 @@ +import { decode, ndjson } from '@brimdata/zed-js'; +// import '../lib/node-globals.js'; +import '../lib/wasm_exec.js'; +import { getInput } from '../lib/input.js'; +import createZedWasm from '../lib/bridge.js'; + +const url = new URL('.', import.meta.url); +const wasmPath = url.href + 'main.wasm'; +const wasm = createZedWasm(fetch(wasmPath).then((r) => r.arrayBuffer())); + +// program?: string, +// input?: string | File | Blob | ReadableStream | any[], +// inputFormat?: LoadFormat, +// outputFormat?: 'js' | 'zed', + +export async function zq(opts) { + const result = await wasm.zq({ + input: getInput(opts.input), + inputFormat: opts.inputFormat, + program: opts.program, + outputFormat: 'zjson', + }); + + const zed = decode(ndjson.parseLines(result)); + if (opts.outputFormat === 'zed') return zed; + return zed.map((val) => val.toJS()); +} + +export function parse(string) { + return wasm.parse(string); +} + +window.zedWasm = { zq, parse }; diff --git a/packages/zed-wasm/src/browser.ts b/packages/zed-wasm/src/browser.ts deleted file mode 100644 index c63326a697..0000000000 --- a/packages/zed-wasm/src/browser.ts +++ /dev/null @@ -1,39 +0,0 @@ -import '../lib/wasm_exec'; -import bridge from 'golang-wasm/src/bridge'; -import { LoadFormat, decode, ndjson } from '@brimdata/zed-js'; -import { getInput } from './input'; - -let go: any; - -async function loadWasm() { - const url = new URL('.', import.meta.url); - const path = url.href + 'main.wasm'; - const wasm = await fetch(path); - console.log('wasm', wasm); - go = bridge(wasm.arrayBuffer()); -} - -export async function zq(opts: { - program?: string; - input?: string | File | Blob | ReadableStream | any[]; - inputFormat?: LoadFormat; - outputFormat?: 'js' | 'zed'; -}) { - if (!go) await loadWasm(); - const result = await go.zq({ - input: getInput(opts.input), - inputFormat: opts.inputFormat, - program: opts.program, - outputFormat: 'zjson', - }); - - const zed = decode(ndjson.parseLines(result)); - if (opts.outputFormat === 'zed') return zed; - return zed.map((val) => val.toJS()); -} - -export async function parse(query: string) { - if (!go) await loadWasm(); - return go.parse(query); -} - diff --git a/packages/zed-wasm/src/golang-wasm.d.ts b/packages/zed-wasm/src/golang-wasm.d.ts deleted file mode 100644 index 6fe715bb19..0000000000 --- a/packages/zed-wasm/src/golang-wasm.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module 'golang-wasm/src/bridge' { - export default function (arg: Promise): any; -} diff --git a/packages/zed-wasm/src/index.js b/packages/zed-wasm/src/index.js new file mode 100644 index 0000000000..9883648d6c --- /dev/null +++ b/packages/zed-wasm/src/index.js @@ -0,0 +1,38 @@ +import { readFile } from 'fs/promises'; +import { decode, ndjson } from '@brimdata/zed-js'; +import '../lib/node-globals.js'; +import '../lib/wasm_exec.js'; +import createZedWasm from '../lib/bridge.js'; +import { getWasmPath } from '../lib/node-wasm-path.js'; + +const wasmFile = getWasmPath(); +const wasm = createZedWasm(readFile(wasmFile)); + +// program?: string, +// input?: string | File | Blob | ReadableStream | any[], +// inputFormat?: LoadFormat, +// outputFormat?: 'js' | 'zed', + +export async function zq(opts) { + const result = await wasm.zq({ + input: getInput(opts.input), + inputFormat: opts.inputFormat, + program: opts.program, + outputFormat: 'zjson', + }); + + const zed = decode(ndjson.parseLines(result)); + if (opts.outputFormat === 'zed') return zed; + return zed.map((val) => val.toJS()); +} + +function getInput(input) { + if (typeof input === 'string') return input; + if (input === undefined) return undefined; + if (input === null) return undefined; + throw new Error(`Unsupported input type provided to zq ${input}`); +} + +export function parse(string) { + return wasm.parse(string); +} diff --git a/packages/zed-wasm/src/index.ts b/packages/zed-wasm/src/index.ts deleted file mode 100644 index 26d0b4f0d6..0000000000 --- a/packages/zed-wasm/src/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -import '../lib/wasm_exec_node'; -import bridge from 'golang-wasm/src/bridge'; -import { LoadFormat, decode, ndjson } from '@brimdata/zed-js'; -import { getInput } from './input'; -import { readFile } from 'fs/promises'; - -let go: any; - -async function loadWasm() { - go = bridge(readFile('./main.wasm')); -} - -export async function zq(opts: { - program?: string; - input?: string | File | Blob | ReadableStream | any[]; - inputFormat?: LoadFormat; - outputFormat?: 'js' | 'zed'; -}) { - if (!go) await loadWasm(); - const result = await go.zq({ - input: getInput(opts.input), - inputFormat: opts.inputFormat, - program: opts.program, - outputFormat: 'zjson', - }); - - const zed = decode(ndjson.parseLines(result)); - if (opts.outputFormat === 'zed') return zed; - return zed.map((val) => val.toJS()); -} - -export async function parse(query: string) { - if (!go) await loadWasm(); - return go.parse(query); -} diff --git a/packages/zed-wasm/test/browser/parse.spec.ts b/packages/zed-wasm/test/browser/parse.spec.ts new file mode 100644 index 0000000000..afe444e863 --- /dev/null +++ b/packages/zed-wasm/test/browser/parse.spec.ts @@ -0,0 +1,15 @@ +import { test, expect } from '@playwright/test'; + +test('page loads', async ({ page }) => { + await page.goto('/test/pages/parse.html'); + await expect( + page.getByRole('heading', { name: 'Parse Test' }) + ).toBeAttached(); +}); + +test('parse a simple zed string', async ({ page }) => { + await page.goto('/test/pages/parse.html'); + await page.fill('input', 'zed := "world"'); + await page.click('button'); + await expect(page.locator('pre')).toContainText('"kind": "OpAssignment"'); +}); diff --git a/packages/zed-wasm/test/browser/zq.test.js b/packages/zed-wasm/test/browser/zq.test.js new file mode 100644 index 0000000000..55f364ae96 --- /dev/null +++ b/packages/zed-wasm/test/browser/zq.test.js @@ -0,0 +1,16 @@ +import { test, expect } from '@playwright/test'; + +test('page loads', async ({ page }) => { + await page.goto('/test/pages/zq.html'); + await expect( + page.getByRole('heading', { name: 'Parse Test' }) + ).toBeAttached(); +}); + +test('run zq', async ({ page }) => { + await page.goto('/test/pages/zq.html'); + await page.fill('[name=input]', '1 2 3'); + await page.fill('[name=script]', 'this * 10'); + await page.click('button'); + await expect(page.locator('pre')).toContainText('[10,20,30]'); +}); diff --git a/packages/zed-wasm/test/cjs/parse.test.js b/packages/zed-wasm/test/cjs/parse.test.js new file mode 100644 index 0000000000..8850a5968d --- /dev/null +++ b/packages/zed-wasm/test/cjs/parse.test.js @@ -0,0 +1,37 @@ +const { parse, zq } = require('@brimdata/zed-wasm'); + +test('parse works in cjs env', async () => { + const resp = await parse('n := "james"'); + expect(resp).toMatchInlineSnapshot(` + { + "ast": [ + { + "assignments": [ + { + "kind": "Assignment", + "lhs": { + "kind": "ID", + "name": "n", + }, + "rhs": { + "kind": "Primitive", + "text": "james", + "type": "string", + }, + }, + ], + "kind": "OpAssignment", + }, + ], + "error": undefined, + } + `); +}); + +test('zq works in cjs env', async () => { + const resp = await zq({ + input: '1 2 3', + program: 'this * 10', + }); + expect(resp).toEqual([10, 20, 30]); +}); diff --git a/packages/zed-wasm/test/node/parser.test.js b/packages/zed-wasm/test/node/parser.test.js new file mode 100644 index 0000000000..d4bcc12154 --- /dev/null +++ b/packages/zed-wasm/test/node/parser.test.js @@ -0,0 +1,29 @@ +import test from 'node:test'; +import assert from 'node:assert'; +import { parse } from '@brimdata/zed-wasm'; + +test('parser', async () => { + const resp = await parse('me := "james"'); + assert.deepEqual(resp, { + error: undefined, + ast: [ + { + assignments: [ + { + kind: 'Assignment', + lhs: { + kind: 'ID', + name: 'me', + }, + rhs: { + kind: 'Primitive', + text: 'james', + type: 'string', + }, + }, + ], + kind: 'OpAssignment', + }, + ], + }); +}); diff --git a/packages/zed-wasm/test/node/zq.test.js b/packages/zed-wasm/test/node/zq.test.js new file mode 100644 index 0000000000..9dac91268d --- /dev/null +++ b/packages/zed-wasm/test/node/zq.test.js @@ -0,0 +1,9 @@ +import test from 'node:test'; +import assert from 'node:assert'; +import { zq } from '@brimdata/zed-wasm'; + +test('input is string', async (t) => { + const input = '1 2 3'; + const resp = await zq({ input, program: 'this + 1' }); + assert.deepEqual(resp, [2, 3, 4]); +}); diff --git a/packages/zed-wasm/test/pages/parse.html b/packages/zed-wasm/test/pages/parse.html new file mode 100644 index 0000000000..117a65c631 --- /dev/null +++ b/packages/zed-wasm/test/pages/parse.html @@ -0,0 +1,50 @@ + + + Parse Test + + + + + + +

Parse Test

+
+ + + +
+ 
+      Output
+
+ + diff --git a/packages/zed-wasm/test/pages/zq.html b/packages/zed-wasm/test/pages/zq.html new file mode 100644 index 0000000000..1e8333952d --- /dev/null +++ b/packages/zed-wasm/test/pages/zq.html @@ -0,0 +1,55 @@ + + + Parse Test + + + + + + +

Parse Test

+
+ + + + + +
+ 
+      Output
+
+ + diff --git a/packages/zed-wasm/test/parse.test.js b/packages/zed-wasm/test/parse.test.js deleted file mode 100644 index 8bffa41ce6..0000000000 --- a/packages/zed-wasm/test/parse.test.js +++ /dev/null @@ -1,13 +0,0 @@ -const assert = chai.assert; - -describe('parse', () => { - it('should parse some shit', async () => { - try { - const resp = await parse('me :='); - console.log('resp', resp) - } - catch (err) { - console.log("error.caught", err) - } - }); -}); diff --git a/packages/zed-wasm/test/zq.test.js b/packages/zed-wasm/test/zq.test.js deleted file mode 100644 index a606c5471f..0000000000 --- a/packages/zed-wasm/test/zq.test.js +++ /dev/null @@ -1,51 +0,0 @@ -const assert = chai.assert; - -describe('zq', () => { - it('input is string', async () => { - const input = '1 2 3'; - const resp = await zq({ input, program: 'this + 1' }); - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('input is file', async () => { - const input = new File(['1 2 3'], 'file.json'); - const resp = await zq({ input, program: 'this + 1' }); - - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('input is blob', async () => { - const input = new Blob(['1 2 3']); - const resp = await zq({ input, program: 'this + 1' }); - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('input is readable stream', async () => { - const input = new Blob(['1 2 3']).stream(); - const resp = await zq({ input, program: 'this + 1' }); - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('input is a fetch', async () => { - const input = await fetch('./package.json'); - const resp = await zq({ input, program: 'yield name' }); - assert.deepEqual(resp, ['@brimdata/zed-wasm']); - }); - - it('input is an array of JS objects', async () => { - const input = [1, 2, 3]; - const zed = await zq({ input, program: 'this + 1', outputFormat: 'zed' }); - const resp = zed.map((val) => val.toJS()); - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('works on 32kb+ file', async () => { - const count = 16000; - let str = ''; - for (let i = 0; i < count; i++) str += `1 `; - const input = new File([str], '32kb.txt'); - const resp = await zq({ input, program: 'count()' }); - - assert.deepEqual(resp, [count]); - }); -}); diff --git a/packages/zed-wasm/tsconfig.json b/packages/zed-wasm/tsconfig.json deleted file mode 100644 index b9a3062984..0000000000 --- a/packages/zed-wasm/tsconfig.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "module": "ES2022", - "moduleResolution": "node", - "target": "ES2017", - "forceConsistentCasingInFileNames": true, - "strict": true, - "noImplicitOverride": true, - "noPropertyAccessFromIndexSignature": true, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true - }, - "files": [], - "include": [], - "references": [ - { - "path": "./tsconfig.lib.json" - }, - { - "path": "./tsconfig.spec.json" - } - ] -} diff --git a/packages/zed-wasm/tsconfig.lib.json b/packages/zed-wasm/tsconfig.lib.json deleted file mode 100644 index 33eca2c2cd..0000000000 --- a/packages/zed-wasm/tsconfig.lib.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "outDir": "../../dist/out-tsc", - "declaration": true, - "types": ["node"] - }, - "include": ["src/**/*.ts"], - "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] -} diff --git a/yarn.lock b/yarn.lock index cd5d087a75..37c7a739b9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1905,6 +1905,16 @@ __metadata: languageName: unknown linkType: soft +"@brimdata/zed-js@file:../../dist/packages/zed-js::locator=%40brimdata%2Fzed-wasm%40workspace%3Apackages%2Fzed-wasm": + version: 0.0.17 + resolution: "@brimdata/zed-js@file:../../dist/packages/zed-js#../../dist/packages/zed-js::hash=26f5d8&locator=%40brimdata%2Fzed-wasm%40workspace%3Apackages%2Fzed-wasm" + dependencies: + event-source-polyfill: ^1.0.31 + events: ^3.3.0 + checksum: fa05674353d07285fab6352a2b13fb182d1d5f766b4608673813eaf3beeb0847b613bd3635fda1180d662d0a947b89c113b726bd1e993a23cf441f8ee5cfec8f + languageName: node + linkType: hard + "@brimdata/zed-js@workspace:*, @brimdata/zed-js@workspace:packages/zed-js": version: 0.0.0-use.local resolution: "@brimdata/zed-js@workspace:packages/zed-js" @@ -1934,9 +1944,16 @@ __metadata: version: 0.0.0-use.local resolution: "@brimdata/zed-wasm@workspace:packages/zed-wasm" dependencies: - "@brimdata/zed-js": "workspace:*" + "@brimdata/zed-js": ../../dist/packages/zed-js + "@playwright/test": ^1.40.1 "@types/golang-wasm": ^1.15.0 + "@types/node": ^20.11.0 + esbuild: ^0.17.12 golang-wasm: "github:teamortix/golang-wasm#master" + jest: ^29.4.1 + npm-run-all: ^4.1.5 + polyfill-crypto.getrandomvalues: ^1.0.0 + serve: ^14.2.1 languageName: unknown linkType: soft @@ -3806,6 +3823,17 @@ __metadata: languageName: node linkType: hard +"@playwright/test@npm:^1.40.1": + version: 1.40.1 + resolution: "@playwright/test@npm:1.40.1" + dependencies: + playwright: 1.40.1 + bin: + playwright: cli.js + checksum: ae094e6cb809365c0707ee2b184e42d2a2542569ada020d2d44ca5866066941262bd9a67af185f86c2fb0133c9b712ea8cb73e2959a289e4261c5fd17077283c + languageName: node + linkType: hard + "@playwright/test@npm:next": version: 1.41.0-alpha-dec-20-2023 resolution: "@playwright/test@npm:1.41.0-alpha-dec-20-2023" @@ -5100,6 +5128,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^20.11.0": + version: 20.11.0 + resolution: "@types/node@npm:20.11.0" + dependencies: + undici-types: ~5.26.4 + checksum: 1bd6890db7e0404d11c33d28f46f19f73256f0ba35d19f0ef2a0faba09f366f188915fb9338eebebcc472075c1c4941e17c7002786aa69afa44980737846b200 + languageName: node + linkType: hard + "@types/normalize-package-data@npm:^2.4.0, @types/normalize-package-data@npm:^2.4.1": version: 2.4.1 resolution: "@types/normalize-package-data@npm:2.4.1" @@ -13735,6 +13772,13 @@ __metadata: languageName: node linkType: hard +"mersenne-twister@npm:^1.0.1": + version: 1.1.0 + resolution: "mersenne-twister@npm:1.1.0" + checksum: 7de1940ded117f2aad9320ae4d21d647b0ecf0667abbadcfe6a2835c669feb674ef46cb7a72da7af69a56d8b19e50e95e2fb7ef6d780efab7a6acd4d87f4cb2d + languageName: node + linkType: hard + "micromark@npm:~2.11.0": version: 2.11.4 resolution: "micromark@npm:2.11.4" @@ -15270,6 +15314,15 @@ __metadata: languageName: node linkType: hard +"playwright-core@npm:1.40.1": + version: 1.40.1 + resolution: "playwright-core@npm:1.40.1" + bin: + playwright-core: cli.js + checksum: 84d92fb9b86e3c225b16b6886bf858eb5059b4e60fa1205ff23336e56a06dcb2eac62650992dede72f406c8e70a7b6a5303e511f9b4bc0b85022ede356a01ee0 + languageName: node + linkType: hard + "playwright-core@npm:1.41.0-alpha-dec-20-2023": version: 1.41.0-alpha-dec-20-2023 resolution: "playwright-core@npm:1.41.0-alpha-dec-20-2023" @@ -15279,6 +15332,21 @@ __metadata: languageName: node linkType: hard +"playwright@npm:1.40.1": + version: 1.40.1 + resolution: "playwright@npm:1.40.1" + dependencies: + fsevents: 2.3.2 + playwright-core: 1.40.1 + dependenciesMeta: + fsevents: + optional: true + bin: + playwright: cli.js + checksum: 9e36791c1b4a649c104aa365fdd9d049924eeb518c5967c0e921aa38b9b00994aa6ee54784d6c2af194b3b494b6f69772673081ef53c6c4a4b2065af9955c4ba + languageName: node + linkType: hard + "playwright@npm:1.41.0-alpha-dec-20-2023, playwright@npm:next": version: 1.41.0-alpha-dec-20-2023 resolution: "playwright@npm:1.41.0-alpha-dec-20-2023" @@ -15313,6 +15381,15 @@ __metadata: languageName: node linkType: hard +"polyfill-crypto.getrandomvalues@npm:^1.0.0": + version: 1.0.0 + resolution: "polyfill-crypto.getrandomvalues@npm:1.0.0" + dependencies: + mersenne-twister: ^1.0.1 + checksum: 73f0880b022af0c5930ef2d34a83a25e03196081082d8ceb204a3e5609a1a3d0dd1656a183073b17bc069a2c5749e44689cebdc4254dae6ed7e4e8b83f451964 + languageName: node + linkType: hard + "postcss-nested@npm:^4.2.1 || ^5.0.0": version: 5.0.6 resolution: "postcss-nested@npm:5.0.6" @@ -16781,6 +16858,27 @@ __metadata: languageName: node linkType: hard +"serve@npm:^14.2.1": + version: 14.2.1 + resolution: "serve@npm:14.2.1" + dependencies: + "@zeit/schemas": 2.29.0 + ajv: 8.11.0 + arg: 5.0.2 + boxen: 7.0.0 + chalk: 5.0.1 + chalk-template: 0.4.0 + clipboardy: 3.0.0 + compression: 1.7.4 + is-port-reachable: 4.0.0 + serve-handler: 6.1.5 + update-check: 1.5.4 + bin: + serve: build/main.js + checksum: c39a517b5d795a0a5c2f9fb9ff088b7e4962c579e34ace5b85dd62f93e0eacbc8a90359792c153c444a83258ffda392113dff7bfd10d41ced574a2d1886c2994 + languageName: node + linkType: hard + "set-blocking@npm:^2.0.0, set-blocking@npm:~2.0.0": version: 2.0.0 resolution: "set-blocking@npm:2.0.0" From 934f640c59cf77d85c33918bd48cba35cbb783c6 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 11 Jan 2024 15:56:20 -0800 Subject: [PATCH 03/43] Zui Tests Passing --- apps/zui/.swcrc | 1 - apps/zui/src/js/models/ast.ts | 18 ++- apps/zui/src/js/models/program.test.ts | 159 +++++++++++----------- apps/zui/src/js/models/program.ts | 82 +++++------ apps/zui/src/js/zed-script/toZedScript.ts | 4 +- apps/zui/tsconfig.json | 1 - packages/zed-js/.swcrc | 31 ----- packages/zed-js/package.json | 3 +- packages/zed-js/project.json | 10 +- packages/zed-wasm/lib/node-globals.js | 5 + packages/zed-wasm/lib/node-wasm-path.js | 15 +- packages/zed-wasm/lib/wasm_exec_node.js | 7 +- packages/zed-wasm/package.json | 7 +- packages/zed-wasm/project.json | 44 +----- tsconfig.base.json | 1 - yarn.lock | 12 +- 16 files changed, 147 insertions(+), 253 deletions(-) delete mode 100644 packages/zed-js/.swcrc diff --git a/apps/zui/.swcrc b/apps/zui/.swcrc index d6f0426068..e15c4688b1 100644 --- a/apps/zui/.swcrc +++ b/apps/zui/.swcrc @@ -14,5 +14,4 @@ "type": "commonjs", "ignoreDynamic": true } - } diff --git a/apps/zui/src/js/models/ast.ts b/apps/zui/src/js/models/ast.ts index 441f5abc38..69f66a5136 100644 --- a/apps/zui/src/js/models/ast.ts +++ b/apps/zui/src/js/models/ast.ts @@ -1,8 +1,18 @@ import {toFieldPath} from "../zed-script/toZedScript" +import {parse} from "@brimdata/zed-wasm" type ColumnName = string | string[] -export default function ast(tree: any) { +export default async function ast(string: string) { + let tree + try { + let res = await parse(string) + if (res.error) throw res.error + tree = res.ast + } catch (error) { + tree = {error} + } + return { valid() { return !tree.error @@ -27,6 +37,12 @@ export default function ast(tree: any) { self() { return tree }, + hasAnalytics() { + for (const proc of this.getProcs()) { + if (ANALYTIC_PROCS.includes(proc.kind)) return true + } + return false + }, } } diff --git a/apps/zui/src/js/models/program.test.ts b/apps/zui/src/js/models/program.test.ts index 5cd003db9d..62a4f1394b 100644 --- a/apps/zui/src/js/models/program.test.ts +++ b/apps/zui/src/js/models/program.test.ts @@ -1,12 +1,12 @@ import {createField, createRecord} from "@brimdata/zed-js" -import program from "./program" +import program, {drillDown, getFilter} from "./program" +import ast from "./ast" describe("excluding and including", () => { const field = createField("uid", "123") test("excluding a field", () => { const script = program('_path=="weird"').exclude(field).string() - expect(script).toEqual('_path=="weird" | uid!="123"') }) @@ -48,83 +48,76 @@ describe("drill down", () => { count: 24, }) - test("when there is no leading filter", () => { - const script = program('count() by this["i d"]["orig h"]') - .drillDown(result) - .string() + test("when there is no leading filter", async () => { + const script = await drillDown('count() by this["i d"]["orig h"]', result) expect(script).toBe('this["i d"]["orig h"]==192.168.0.54') }) - test("when there is a sort on there", () => { - const script = program('name=="james" | count() by proto | sort -r count') - .drillDown(result) - .string() + test("when there is a sort on there", async () => { + const script = await drillDown( + 'name=="james" | count() by proto | sort -r count', + result + ) expect(script).toBe('name=="james" | proto=="udp"') }) - test("when there is a grep with a star", () => { - const script = program( - 'grep(/(*|Elm)/) Category=="Furnishings" | count() by proto' + test("when there is a grep with a star", async () => { + const script = await drillDown( + 'grep(/(*|Elm)/) Category=="Furnishings" | count() by proto', + result ) - .drillDown(result) - .string() expect(script).toBe( 'grep(/(*|Elm)/) Category=="Furnishings" | proto=="udp"' ) }) - test("combines keys in the group by proc", () => { - const script = program( - '_path=="dns" | count() by id.orig_h, proto, query | sort -r' + test("combines keys in the group by proc", async () => { + const script = await drillDown( + '_path=="dns" | count() by id.orig_h, proto, query | sort -r', + result ) - .drillDown(result) - .string() expect(script).toBe( '_path=="dns" | id.orig_h==192.168.0.54 proto=="udp" query=="WPAD"' ) }) - test("removes *", () => { - const script = program("* | count() by id.orig_h") - .drillDown(result) - .string() + test("removes *", async () => { + const script = await drillDown("* | count() by id.orig_h", result) expect(script).toBe("id.orig_h==192.168.0.54") }) - test("easy peasy", () => { - const script = program("names james | count() by proto") - .drillDown(result) - .string() + test("easy peasy", async () => { + const script = await drillDown("names james | count() by proto", result) expect(script).toBe('names james | proto=="udp"') }) - test("count by and filter the same", () => { + test("count by and filter the same", async () => { const result = createRecord({md5: "123", count: 1}) - const script = program('md5=="123" | count() by md5 | sort -r | head 5') - .drillDown(result) - .string() + const script = await drillDown( + 'md5=="123" | count() by md5 | sort -r | head 5', + result + ) expect(script).toEqual('md5=="123"') }) - test("filter query", () => { + test("filter query", async () => { const result = createRecord({ md5: "9f51ef98c42df4430a978e4157c43dd5", count: 21, }) - const script = program( - '_path=="files" filename!="-" | count() by md5,filename | count() by md5 | sort -r | filter count > 1' + const script = await drillDown( + '_path=="files" filename!="-" | count() by md5,filename | count() by md5 | sort -r | filter count > 1', + result ) - .drillDown(result) - .string() expect(script).toEqual( '_path=="files" filename!="-" | md5=="9f51ef98c42df4430a978e4157c43dd5"' @@ -178,91 +171,95 @@ describe("sort by", () => { }) }) +jest.setTimeout(30_000) describe("#hasAnalytics()", () => { - test("head proc does not have analytics", () => { - expect(program("* | head 2").hasAnalytics()).toBe(false) + test("head proc does not have analytics", async () => { + const tree = await ast("* | head 2") + expect(tree.hasAnalytics()).toBe(false) }) - test("sort proc does not have analytics", () => { - expect(program("* | sort -r id.resp_p").hasAnalytics()).toBe(false) + test("sort proc does not have analytics", async () => { + const tree = await ast("* | sort -r id.resp_p") + expect(tree.hasAnalytics()).toBe(false) }) - test("every proc does contain analytics", () => { - expect(program("* | count() by every(1h)").hasAnalytics()).toBe(true) + test("every proc does contain analytics", async () => { + const tree = await ast("* | count() by every(1h)") + expect(tree.hasAnalytics()).toBe(true) }) - test("parallel procs when one does have analytics", () => { - expect( - program( - "* | fork ( => count() by every(1h) => count() by id.resp_h )" - ).hasAnalytics() - ).toBe(true) + test("parallel procs when one does have analytics", async () => { + const tree = await ast( + "* | fork ( => count() by every(1h) => count() by id.resp_h )" + ) + expect(tree.hasAnalytics()).toBe(true) }) - test("parallel procs when both do not have analytics", () => { - expect(program("* | head 100; head 200").hasAnalytics()).toBe(false) + test("parallel procs when both do not have analytics", async () => { + const tree = await ast("* | head 100; head 200") + expect(tree.hasAnalytics()).toBe(false) }) - test("when there are no procs", () => { - expect(program("*").hasAnalytics()).toBe(false) + test("when there are no procs", async () => { + const tree = await ast("*") + expect(tree.hasAnalytics()).toBe(false) }) - test("for a crappy string", () => { - expect(program("-r").hasAnalytics()).toBe(false) + test("for a crappy string", async () => { + const tree = await ast("-r") + expect(tree.hasAnalytics()).toBe(false) }) - test("for sequential proc", () => { - expect( - program("*google* | head 3 | sort -r id.resp_p").hasAnalytics() - ).toBe(false) + test("for sequential proc", async () => { + const tree = await ast("*google* | head 3 | sort -r id.resp_p") + expect(tree.hasAnalytics()).toBe(false) }) - test("for cut proc", () => { - expect( - program( - "* | fork ( => cut uid, _path => cut uid ) | tail 1" - ).hasAnalytics() - ).toBe(false) + test("for cut proc", async () => { + const tree = await ast("* | fork ( => cut uid, _path => cut uid ) | tail 1") + expect(tree.hasAnalytics()).toBe(false) }) - test("for filter proc", () => { - expect(program('* | filter _path=="conn"').hasAnalytics()).toBe(false) + test("for filter proc", async () => { + const tree = await ast('* | filter _path=="conn"') + expect(tree.hasAnalytics()).toBe(false) }) }) describe("extracting the first filter", () => { - test("*", () => { - expect(program("*").filter()).toEqual("*") + test("*", async () => { + expect(await getFilter("*")).toEqual("*") }) - test('_path=="conn"', () => { - expect(program('_path=="conn"').filter()).toEqual('_path=="conn"') + test('_path=="conn"', async () => { + expect(await getFilter('_path=="conn"')).toEqual('_path=="conn"') }) - test('_path=="conn" | sum(duration)', () => { - expect(program('_path=="conn" | sum(duration)').filter()).toEqual( + test('_path=="conn" | sum(duration)', async () => { + expect(await getFilter('_path=="conn" | sum(duration)')).toEqual( '_path=="conn"' ) }) - test('_path=="conn" | filter a', () => { - expect(program('_path=="conn" | filter a').filter()).toEqual( + test('_path=="conn" | filter a', async () => { + expect(await getFilter('_path=="conn" | filter a')).toEqual( '_path=="conn" | filter a' ) }) - test("count()", () => { - expect(program("count()").filter()).toEqual("*") + test("count()", async () => { + expect(await getFilter("count()")).toEqual("*") }) - test("dns | count() | filter num > 1", () => { - expect(program("dns | count() | filter num > 1").filter()).toEqual("dns") + test("dns | count() | filter num > 1", async () => { + expect(await getFilter("dns | count() | filter num > 1")).toEqual("dns") }) }) describe("cut", () => { - test("cut some fields", () => { - expect(program("my filter").cut("ts", "_path").string()).toBe( + test("cut some fields", async () => { + const filter = await getFilter("my filter") + expect(program(filter).cut("ts", "_path").string()).toBe( "my filter | cut ts, _path" ) }) diff --git a/apps/zui/src/js/models/program.ts b/apps/zui/src/js/models/program.ts index 41fcab7484..a6d95b34e6 100644 --- a/apps/zui/src/js/models/program.ts +++ b/apps/zui/src/js/models/program.ts @@ -1,11 +1,10 @@ import * as zed from "@brimdata/zed-js" -import {parse as parseAst} from "@brimdata/zed-wasm" import {isEmpty, last} from "lodash" import {trim} from "../lib/Str" -import ast, {ANALYTIC_PROCS} from "./ast" +import ast from "./ast" import syntax from "./syntax" -export default function (p = "") { +export default function program(p = "") { return { exclude(field: zed.Field) { p = appendWithPipe(p, syntax.exclude(field)) @@ -36,23 +35,6 @@ export default function (p = "") { return this.cut(fields.map((fieldName) => "quiet(" + fieldName + ")")) }, - drillDown(log: zed.Record) { - let filter = this.filter() - - const newFilters = this.ast() - .groupByKeys() - .map((name) => log.tryField(name)) - .filter((f) => !!f) - .map(syntax.include) - .join(" ") - - if (/^\s*\*\s*$/.test(filter)) filter = "" - if (newFilters.includes(filter)) filter = "" - - p = appendWithPipe(filter, newFilters) - return this - }, - countBy(name: string | string[]) { p = appendWithPipe(p, syntax.countBy(name)) return this @@ -64,31 +46,6 @@ export default function (p = "") { return this }, - async ast() { - let tree - try { - let res = await parseAst(p) - if (res.error) throw res.error - tree = res.ast - } catch (error) { - tree = {error} - } - return ast(tree) - }, - - filter() { - const [head, ...tail] = p.split( - /\|?\s*(summarize|count|countdistinct|sum)/i - ) - - if (isEmpty(tail) && this.hasAnalytics()) { - return "*" - } else { - if (isEmpty(trim(head))) return "*" - return trim(head) - } - }, - procs() { const [_, ...procs] = p.split("|") return procs.join("|") @@ -97,16 +54,39 @@ export default function (p = "") { string() { return p.trim() === "" ? "*" : p }, + } +} - hasAnalytics() { - for (const proc of this.ast().getProcs()) { - if (ANALYTIC_PROCS.includes(proc.kind)) return true - } - return false - }, +export async function getFilter(string) { + const [head, ...tail] = string.split( + /\|?\s*(summarize|count|countdistinct|sum)/i + ) + const tree = await ast(string) + if (isEmpty(tail) && tree.hasAnalytics()) { + return "*" + } else { + if (isEmpty(trim(head))) return "*" + return trim(head) } } +export async function drillDown(script: string, log: zed.Record) { + let filter = await getFilter(script) + + const newFilters = (await ast(script)) + .groupByKeys() + .map((name) => log.tryField(name)) + .filter((f) => !!f) + .map(syntax.include) + .join(" ") + + if (/^\s*\*\s*$/.test(filter)) filter = "" + if (newFilters.includes(filter)) filter = "" + + script = appendWithPipe(filter, newFilters) + return script +} + function appendWithPipe(program, filter) { if (isEmpty(program)) return filter if (!isWhitespace(last(program))) program += " " diff --git a/apps/zui/src/js/zed-script/toZedScript.ts b/apps/zui/src/js/zed-script/toZedScript.ts index d37b8560c2..7422fa0215 100644 --- a/apps/zui/src/js/zed-script/toZedScript.ts +++ b/apps/zui/src/js/zed-script/toZedScript.ts @@ -29,7 +29,9 @@ export function toZedScript(object: unknown): string { if (object instanceof Date) return toZedScriptDate(object) if (typeof object === "boolean") return toZedScriptBool(object) if (object === null) return toZedScriptNull() - throw new Error(`Can't convert object to Zed script: ${object}`) + throw new Error( + `Can't convert object to Zed script: ${JSON.stringify(object)}` + ) } const DOUBLE_QUOTE = /"/g diff --git a/apps/zui/tsconfig.json b/apps/zui/tsconfig.json index 5feb82e22a..8e0d967770 100644 --- a/apps/zui/tsconfig.json +++ b/apps/zui/tsconfig.json @@ -21,7 +21,6 @@ "paths": { "@brimdata/zed-js": ["../../packages/zed-js/src/index.ts"], "@brimdata/zed-node": ["../../packages/zed-node/src/index.ts"], - "@brimdata/zed-wasm": ["../../packages/zed-wasm/src/index.ts"], "zui-test-data": ["../../packages/zui-test-data/src/index.ts"], } }, diff --git a/packages/zed-js/.swcrc b/packages/zed-js/.swcrc deleted file mode 100644 index fb42f4b043..0000000000 --- a/packages/zed-js/.swcrc +++ /dev/null @@ -1,31 +0,0 @@ -{ - "jsc": { - "target": "es2017", - "parser": { - "syntax": "typescript", - "decorators": true, - "dynamicImport": true - }, - "transform": { - "decoratorMetadata": true, - "legacyDecorator": true - }, - "keepClassNames": true, - "externalHelpers": true, - "loose": true - }, - "module": { - "type": "commonjs", - "strict": true, - "noInterop": true - }, - "sourceMaps": true, - "exclude": [ - "jest.config.ts", - ".*\\.spec.tsx?$", - ".*\\.test.tsx?$", - "./src/jest-setup.ts$", - "./**/jest-setup.ts$", - ".*.js$" - ] -} diff --git a/packages/zed-js/package.json b/packages/zed-js/package.json index 4ccc33af26..33a3ae1a4d 100644 --- a/packages/zed-js/package.json +++ b/packages/zed-js/package.json @@ -2,7 +2,8 @@ "name": "@brimdata/zed-js", "version": "0.0.17", "type": "commonjs", - "exports": "./src/index.js", + "main": "./dist/cjs/src/index.js", + "jsdelivr": "./dist/browser/index.js", "devDependencies": { "@types/event-source-polyfill": "^1.0.1" }, diff --git a/packages/zed-js/project.json b/packages/zed-js/project.json index e66d8d5662..8be5749645 100644 --- a/packages/zed-js/project.json +++ b/packages/zed-js/project.json @@ -6,21 +6,17 @@ "targets": { "build": { "executor": "nx:run-commands", - "options": { - "outputPath": "dist/packages/zed-js", - "commands": [] - }, + "options": { "commands": [] }, "dependsOn": ["build-node", "build-browser"] }, "build-node": { "executor": "@nrwl/js:tsc", "outputs": ["{options.outputPath}"], "options": { - "outputPath": "dist/packages/zed-js", + "outputPath": "packages/zed-js/dist/cjs", "main": "packages/zed-js/src/index.ts", "tsConfig": "packages/zed-js/tsconfig.lib.json", "assets": ["packages/zed-js/*.md"], - "buildableProjectDepsInPackageJsonType": "dependencies", "clean": false } }, @@ -28,7 +24,7 @@ "executor": "@nrwl/esbuild:esbuild", "outputs": ["{options.outputPath}"], "options": { - "outputPath": "dist/packages/zed-js", + "outputPath": "packages/zed-js/dist/browser", "bundle": true, "minify": true, "format": ["esm"], diff --git a/packages/zed-wasm/lib/node-globals.js b/packages/zed-wasm/lib/node-globals.js index ffe6482b09..869b2aa939 100644 --- a/packages/zed-wasm/lib/node-globals.js +++ b/packages/zed-wasm/lib/node-globals.js @@ -3,3 +3,8 @@ import getRandomValues from 'polyfill-crypto.getrandomvalues'; globalThis.crypto = { ...crypto, getRandomValues }; globalThis.requestAnimationFrame = setTimeout; +globalThis.require = require; +globalThis.fs = require('fs'); +globalThis.TextEncoder = require('util').TextEncoder; +globalThis.TextDecoder = require('util').TextDecoder; +globalThis.performance ??= require('performance'); diff --git a/packages/zed-wasm/lib/node-wasm-path.js b/packages/zed-wasm/lib/node-wasm-path.js index 583ba58c57..9d2205f48e 100644 --- a/packages/zed-wasm/lib/node-wasm-path.js +++ b/packages/zed-wasm/lib/node-wasm-path.js @@ -1,14 +1,7 @@ -import { join, dirname } from 'path'; -import { fileURLToPath } from 'url'; +import { join } from 'path'; export function getWasmPath() { - if (import.meta.url) { - // In ESM we go back to the dist - const dir = dirname(fileURLToPath(import.meta.url)); - return join(dir, '../dist/main.wasm'); - } else { - // In CSJ we are already in dist because we got built - const dir = __dirname; - return join(dir, './main.wasm'); - } + // In CSJ we are already in dist because we got built + const dir = __dirname; + return join(dir, './main.wasm'); } diff --git a/packages/zed-wasm/lib/wasm_exec_node.js b/packages/zed-wasm/lib/wasm_exec_node.js index 93d9654ac2..4d6ee31195 100644 --- a/packages/zed-wasm/lib/wasm_exec_node.js +++ b/packages/zed-wasm/lib/wasm_exec_node.js @@ -1,9 +1,4 @@ -global.require = require; -global.fs = require('fs'); -global.TextEncoder = require('util').TextEncoder; -global.TextDecoder = require('util').TextDecoder; -global.performance ??= require('performance'); -global.crypto ??= require('crypto'); + const path = require('path'); require('./wasm_exec'); diff --git a/packages/zed-wasm/package.json b/packages/zed-wasm/package.json index 494bfbc535..3856f5240b 100644 --- a/packages/zed-wasm/package.json +++ b/packages/zed-wasm/package.json @@ -10,11 +10,6 @@ "src/**/*.js" ], "main": "./dist/index.cjs", - "module": "./src/index.js", - "exports": { - "require": "./dist/index.cjs", - "default": "./src/index.js" - }, "jsdelivr": "./dist/browser.js", "scripts": { "test": "run-s 'test:*'", @@ -45,7 +40,7 @@ "serve": "^14.2.1" }, "dependencies": { - "@brimdata/zed-js": "../../dist/packages/zed-js", + "@brimdata/zed-js": "workspace:*", "polyfill-crypto.getrandomvalues": "^1.0.0" } } diff --git a/packages/zed-wasm/project.json b/packages/zed-wasm/project.json index 6d35f9f512..48e71b0365 100644 --- a/packages/zed-wasm/project.json +++ b/packages/zed-wasm/project.json @@ -2,47 +2,5 @@ "name": "zed-wasm", "$schema": "../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "packages/zed-wasm/src", - "projectType": "library", - "targets": { - "build": { - "executor": "nx:run-commands", - "options": { - "outputPath": "dist/packages/zed-js", - "commands": [] - }, - "dependsOn": ["build-node", "build-browser"] - }, - "build-go": { - "executor": "nx:run-commands", - "outputs": ["dist/packages/zed-wasm/main.wasm"], - "options": { - "command": "go build -tags=noasm -o ../../dist/packages/zed-wasm/main.wasm main.go", - "cwd": "packages/zed-wasm" - } - }, - "publish": { - "executor": "nx:run-commands", - "options": { - "command": "node tools/scripts/publish.mjs zed-wasm {args.ver} {args.tag}" - }, - "dependsOn": ["build"] - }, - "lint": { - "executor": "@nrwl/linter:eslint", - "outputs": ["{options.outputFile}"], - "options": { - "lintFilePatterns": ["packages/zed-wasm/**/*.ts"] - } - }, - "test-browser": { - "executor": "nx:run-commands", - "options": { - "commands": [ - "serve", - "open-cli http://localhost:3000/packages/zed-wasm/" - ] - } - } - }, - "tags": [] + "projectType": "library" } diff --git a/tsconfig.base.json b/tsconfig.base.json index 29e3275df4..408a5545ed 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -18,7 +18,6 @@ "@brimdata/sample-data": ["packages/sample-data/src/index.ts"], "@brimdata/zed-js": ["packages/zed-js/src/index.ts"], "@brimdata/zed-node": ["packages/zed-node/src/index.ts"], - "@brimdata/zed-wasm": ["packages/zed-wasm/src/index.ts"], "zui-test-data": ["packages/zui-test-data/src/index.ts"] } }, diff --git a/yarn.lock b/yarn.lock index 37c7a739b9..2e05d63372 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1905,16 +1905,6 @@ __metadata: languageName: unknown linkType: soft -"@brimdata/zed-js@file:../../dist/packages/zed-js::locator=%40brimdata%2Fzed-wasm%40workspace%3Apackages%2Fzed-wasm": - version: 0.0.17 - resolution: "@brimdata/zed-js@file:../../dist/packages/zed-js#../../dist/packages/zed-js::hash=26f5d8&locator=%40brimdata%2Fzed-wasm%40workspace%3Apackages%2Fzed-wasm" - dependencies: - event-source-polyfill: ^1.0.31 - events: ^3.3.0 - checksum: fa05674353d07285fab6352a2b13fb182d1d5f766b4608673813eaf3beeb0847b613bd3635fda1180d662d0a947b89c113b726bd1e993a23cf441f8ee5cfec8f - languageName: node - linkType: hard - "@brimdata/zed-js@workspace:*, @brimdata/zed-js@workspace:packages/zed-js": version: 0.0.0-use.local resolution: "@brimdata/zed-js@workspace:packages/zed-js" @@ -1944,7 +1934,7 @@ __metadata: version: 0.0.0-use.local resolution: "@brimdata/zed-wasm@workspace:packages/zed-wasm" dependencies: - "@brimdata/zed-js": ../../dist/packages/zed-js + "@brimdata/zed-js": "workspace:*" "@playwright/test": ^1.40.1 "@types/golang-wasm": ^1.15.0 "@types/node": ^20.11.0 From 39b030ff6d2777e7d73bf77565c4ab3fbb2b9452 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 16 Jan 2024 14:37:52 -0800 Subject: [PATCH 04/43] Zed Wasm Complete --- packages/zed-wasm/.eslintrc.json | 21 ---- packages/zed-wasm/lib/bridge.js | 121 +------------------ packages/zed-wasm/lib/input.js | 23 ---- packages/zed-wasm/lib/node-globals.js | 1 - packages/zed-wasm/lib/node-wasm-path.js | 7 -- packages/zed-wasm/lib/wasm_exec.d.ts | 0 packages/zed-wasm/lib/wasm_exec_node.js | 17 --- packages/zed-wasm/src/browser.js | 58 ++++----- packages/zed-wasm/src/index.js | 38 ++---- packages/zed-wasm/src/interface.js | 24 ++++ packages/zed-wasm/test/browser/parse.spec.ts | 7 +- packages/zed-wasm/test/browser/zq.test.js | 7 +- packages/zed-wasm/test/cjs/parse.test.js | 13 +- packages/zed-wasm/test/node/parser.test.js | 10 +- packages/zed-wasm/test/node/zq.test.js | 10 +- packages/zed-wasm/test/pages/parse.html | 7 +- packages/zed-wasm/test/pages/zq.html | 7 +- 17 files changed, 114 insertions(+), 257 deletions(-) delete mode 100644 packages/zed-wasm/.eslintrc.json delete mode 100644 packages/zed-wasm/lib/input.js delete mode 100644 packages/zed-wasm/lib/node-wasm-path.js delete mode 100644 packages/zed-wasm/lib/wasm_exec.d.ts delete mode 100644 packages/zed-wasm/lib/wasm_exec_node.js create mode 100644 packages/zed-wasm/src/interface.js diff --git a/packages/zed-wasm/.eslintrc.json b/packages/zed-wasm/.eslintrc.json deleted file mode 100644 index c6f0c80ab9..0000000000 --- a/packages/zed-wasm/.eslintrc.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "extends": ["../../.eslintrc.json"], - "ignorePatterns": ["!**/*"], - "overrides": [ - { - "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], - "rules": {} - }, - { - "files": ["*.ts", "*.tsx"], - "rules": { - "@typescript-eslint/ban-ts-comment": "off", - "@typescript-eslint/no-explicit-any": "off" - } - }, - { - "files": ["*.js", "*.jsx"], - "rules": {} - } - ] -} diff --git a/packages/zed-wasm/lib/bridge.js b/packages/zed-wasm/lib/bridge.js index a340abadc7..52b6a38e75 100644 --- a/packages/zed-wasm/lib/bridge.js +++ b/packages/zed-wasm/lib/bridge.js @@ -1,32 +1,3 @@ -// Initially, the __go_wasm__ object will be an empty object. -const g = globalThis; -if (!g.__go_wasm__) { - g.__go_wasm__ = {}; -} - -/** - * The maximum amount of time that we would expect Wasm to take to initialize. - * If it doesn't initialize after this time, we send a warning to console. - * Most likely something has gone wrong if it takes more than 3 seconds to initialize. - */ -const maxTime = 3 * 1000; - -/** - * bridge is an easier way to refer to the Go WASM object. - */ -const bridge = g.__go_wasm__; - -/** - * Wrapper is used by Go to run all Go functions in JS. - * - * @param {Function} goFunc a function that is expected to return an object of the following specification: - * { - * result: undefined | any // undefined when error is returned, or function returns undefined. - * error: Error | undefined // undefined when no error is present. - * } - * - * @returns {Function} returns a function that take arguments which are used to call the Go function. - */ function wrapper(goFunc) { return (...args) => { const result = goFunc.apply(undefined, args); @@ -37,91 +8,7 @@ function wrapper(goFunc) { }; } -/** - * Sleep is used when awaiting for Go Wasm to initialize. - * It uses the lowest possible sane delay time (via requestAnimationFrame). - * However, if the window is not focused, requestAnimationFrame never returns. - * A timeout will ensure to be called after 50 ms, regardless of whether or not the tab is in focus. - * - * @returns {Promise} an always-resolving promise when a tick has been completed. - */ -function sleep() { - return new Promise((res) => { - requestAnimationFrame(() => res()); - setTimeout(() => { - res(); - }, 50); - }); -} - -/** - * @param {ArrayBuffer} getBytes a promise that is bytes of the Go Wasm object. - * - * @returns {Proxy} an object that can be used to call WASM's objects and properly parse their results. - * - * All values that want to be retrieved from the proxy, regardless of if they are a function or not, must be retrieved - * as if they are from a function call. - * - * If a non-function value is returned however arguments are provided, a warning will be printed. - */ -export default function (getBytes) { - let proxy; - let go; - - async function init() { - bridge.__wrapper__ = wrapper; - - go = new g.Go(); - let bytes = await getBytes; - let result = await WebAssembly.instantiate(bytes, go.importObject); - go.run(result.instance); - } - - init(); - const id = setTimeout(() => { - if (bridge.__ready__ !== true) { - console.warn( - 'Golang WASM Bridge (__go_wasm__.__ready__) still not true after max time' - ); - } - }, maxTime); - - proxy = new Proxy( - {}, - { - get: (_, key) => { - return (...args) => { - return new Promise(async (res, rej) => { - if (!go || go.exited) { - return rej(new Error('The Go instance is not active.')); - } - while (bridge.__ready__ !== true) { - await sleep(); - } - clearTimeout(id); - - if (typeof bridge[key] !== 'function') { - res(bridge[key]); - - if (args.length !== 0) { - console.warn( - 'Retrieved value from WASM returned function type, however called with arguments.' - ); - } - return; - } - - try { - res(bridge[key].apply(undefined, args)); - } catch (e) { - rej(e); - } - }); - }; - }, - } - ); - - bridge.__proxy__ = proxy; - return proxy; -} +globalThis.__go_wasm__ = { + __wrapper__: wrapper, + __ready__: false, +}; diff --git a/packages/zed-wasm/lib/input.js b/packages/zed-wasm/lib/input.js deleted file mode 100644 index 9d62cc2380..0000000000 --- a/packages/zed-wasm/lib/input.js +++ /dev/null @@ -1,23 +0,0 @@ -export function getInput(input) { - if (typeof input === 'string') return input; - if (input instanceof File) return input.stream(); - if (input instanceof Blob) return input.stream(); - if (input instanceof ReadableStream) return input; - if (input instanceof Response) return input.body; - if (Array.isArray(input)) return arrayStream(input); - if (input === undefined) return undefined; - if (input === null) return undefined; - throw new Error(`Unsupported input type provided to zq ${input}`); -} - -function arrayStream(input) { - const encoder = new TextEncoder(); - return new ReadableStream({ - start(ctl) { - for (const item of input) { - ctl.enqueue(encoder.encode(JSON.stringify(item) + '\n')); - } - ctl.close(); - }, - }); -} diff --git a/packages/zed-wasm/lib/node-globals.js b/packages/zed-wasm/lib/node-globals.js index 869b2aa939..43ec7632e4 100644 --- a/packages/zed-wasm/lib/node-globals.js +++ b/packages/zed-wasm/lib/node-globals.js @@ -7,4 +7,3 @@ globalThis.require = require; globalThis.fs = require('fs'); globalThis.TextEncoder = require('util').TextEncoder; globalThis.TextDecoder = require('util').TextDecoder; -globalThis.performance ??= require('performance'); diff --git a/packages/zed-wasm/lib/node-wasm-path.js b/packages/zed-wasm/lib/node-wasm-path.js deleted file mode 100644 index 9d2205f48e..0000000000 --- a/packages/zed-wasm/lib/node-wasm-path.js +++ /dev/null @@ -1,7 +0,0 @@ -import { join } from 'path'; - -export function getWasmPath() { - // In CSJ we are already in dist because we got built - const dir = __dirname; - return join(dir, './main.wasm'); -} diff --git a/packages/zed-wasm/lib/wasm_exec.d.ts b/packages/zed-wasm/lib/wasm_exec.d.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/zed-wasm/lib/wasm_exec_node.js b/packages/zed-wasm/lib/wasm_exec_node.js deleted file mode 100644 index 4d6ee31195..0000000000 --- a/packages/zed-wasm/lib/wasm_exec_node.js +++ /dev/null @@ -1,17 +0,0 @@ - -const path = require('path'); -require('./wasm_exec'); - -const go = new Go(); -go.argv = process.argv.slice(2); -go.env = Object.assign({ TMPDIR: require('os').tmpdir() }, process.env); -go.exit = process.exit; - -const wasmFile = path.join(__dirname, '../dist/main.wasm'); -async function initZedWasm() { - const buffer = global.fs.readFileSync(wasmFile); - const result = await WebAssembly.instantiate(buffer, go.importObject); - return go.run(result.instance); -} - -module.exports = { initZedWasm }; diff --git a/packages/zed-wasm/src/browser.js b/packages/zed-wasm/src/browser.js index d5a516add3..ae748f4f65 100644 --- a/packages/zed-wasm/src/browser.js +++ b/packages/zed-wasm/src/browser.js @@ -1,33 +1,37 @@ -import { decode, ndjson } from '@brimdata/zed-js'; -// import '../lib/node-globals.js'; import '../lib/wasm_exec.js'; -import { getInput } from '../lib/input.js'; -import createZedWasm from '../lib/bridge.js'; +import '../lib/bridge.js'; +import { createInterface } from './interface.js'; -const url = new URL('.', import.meta.url); -const wasmPath = url.href + 'main.wasm'; -const wasm = createZedWasm(fetch(wasmPath).then((r) => r.arrayBuffer())); - -// program?: string, -// input?: string | File | Blob | ReadableStream | any[], -// inputFormat?: LoadFormat, -// outputFormat?: 'js' | 'zed', - -export async function zq(opts) { - const result = await wasm.zq({ - input: getInput(opts.input), - inputFormat: opts.inputFormat, - program: opts.program, - outputFormat: 'zjson', - }); - - const zed = decode(ndjson.parseLines(result)); - if (opts.outputFormat === 'zed') return zed; - return zed.map((val) => val.toJS()); +export async function initZedWasm(fetchRequest) { + const go = new Go(); + const { instance } = await WebAssembly.instantiateStreaming( + fetchRequest, + go.importObject + ); + go.run(instance); + return createInterface(__go_wasm__, { getInput }); } -export function parse(string) { - return wasm.parse(string); +function getInput(input) { + if (typeof input === 'string') return input; + if (input instanceof File) return input.stream(); + if (input instanceof Blob) return input.stream(); + if (input instanceof ReadableStream) return input; + if (input instanceof Response) return input.body; + if (Array.isArray(input)) return arrayStream(input); + if (input === undefined) return undefined; + if (input === null) return undefined; + throw new Error(`Unsupported input type provided to zq ${input}`); } -window.zedWasm = { zq, parse }; +function arrayStream(input) { + const encoder = new TextEncoder(); + return new ReadableStream({ + start(ctl) { + for (const item of input) { + ctl.enqueue(encoder.encode(JSON.stringify(item) + '\n')); + } + ctl.close(); + }, + }); +} diff --git a/packages/zed-wasm/src/index.js b/packages/zed-wasm/src/index.js index 9883648d6c..51149aa9df 100644 --- a/packages/zed-wasm/src/index.js +++ b/packages/zed-wasm/src/index.js @@ -1,29 +1,17 @@ -import { readFile } from 'fs/promises'; -import { decode, ndjson } from '@brimdata/zed-js'; import '../lib/node-globals.js'; import '../lib/wasm_exec.js'; -import createZedWasm from '../lib/bridge.js'; -import { getWasmPath } from '../lib/node-wasm-path.js'; +import '../lib/bridge.js'; +import { readFile } from 'node:fs/promises'; +import { createInterface } from './interface.js'; -const wasmFile = getWasmPath(); -const wasm = createZedWasm(readFile(wasmFile)); - -// program?: string, -// input?: string | File | Blob | ReadableStream | any[], -// inputFormat?: LoadFormat, -// outputFormat?: 'js' | 'zed', - -export async function zq(opts) { - const result = await wasm.zq({ - input: getInput(opts.input), - inputFormat: opts.inputFormat, - program: opts.program, - outputFormat: 'zjson', - }); - - const zed = decode(ndjson.parseLines(result)); - if (opts.outputFormat === 'zed') return zed; - return zed.map((val) => val.toJS()); +export async function initZedWasm(wasmFilePath) { + const go = new Go(); + const { instance } = await WebAssembly.instantiate( + await readFile(wasmFilePath), + go.importObject + ); + go.run(instance); + return createInterface(__go_wasm__, { getInput }); } function getInput(input) { @@ -32,7 +20,3 @@ function getInput(input) { if (input === null) return undefined; throw new Error(`Unsupported input type provided to zq ${input}`); } - -export function parse(string) { - return wasm.parse(string); -} diff --git a/packages/zed-wasm/src/interface.js b/packages/zed-wasm/src/interface.js new file mode 100644 index 0000000000..c67ca1920d --- /dev/null +++ b/packages/zed-wasm/src/interface.js @@ -0,0 +1,24 @@ +import { decode, ndjson } from '@brimdata/zed-js'; + +export function createInterface(go, deps) { + return { + // program?: string, + // input?: string | File | Blob | ReadableStream | any[], + // inputFormat?: LoadFormat, + // outputFormat?: 'js' | 'zed', + zq: async (opts) => { + const result = await go.zq({ + input: deps.getInput(opts.input), + inputFormat: opts.inputFormat, + program: opts.program, + outputFormat: 'zjson', + }); + + const zed = decode(ndjson.parseLines(result)); + if (opts.outputFormat === 'zed') return zed; + return zed.map((val) => val.toJS()); + }, + + parse: (string) => go.parse(string), + }; +} diff --git a/packages/zed-wasm/test/browser/parse.spec.ts b/packages/zed-wasm/test/browser/parse.spec.ts index afe444e863..266ae08a96 100644 --- a/packages/zed-wasm/test/browser/parse.spec.ts +++ b/packages/zed-wasm/test/browser/parse.spec.ts @@ -1,14 +1,11 @@ import { test, expect } from '@playwright/test'; -test('page loads', async ({ page }) => { +test.beforeEach(async ({ page }) => { await page.goto('/test/pages/parse.html'); - await expect( - page.getByRole('heading', { name: 'Parse Test' }) - ).toBeAttached(); + await page.getByText('Wasm Ready').waitFor({ state: 'attached' }); }); test('parse a simple zed string', async ({ page }) => { - await page.goto('/test/pages/parse.html'); await page.fill('input', 'zed := "world"'); await page.click('button'); await expect(page.locator('pre')).toContainText('"kind": "OpAssignment"'); diff --git a/packages/zed-wasm/test/browser/zq.test.js b/packages/zed-wasm/test/browser/zq.test.js index 55f364ae96..ef1e3c58ae 100644 --- a/packages/zed-wasm/test/browser/zq.test.js +++ b/packages/zed-wasm/test/browser/zq.test.js @@ -1,14 +1,11 @@ import { test, expect } from '@playwright/test'; -test('page loads', async ({ page }) => { +test.beforeEach(async ({ page }) => { await page.goto('/test/pages/zq.html'); - await expect( - page.getByRole('heading', { name: 'Parse Test' }) - ).toBeAttached(); + await page.getByText('Wasm Ready').waitFor({ state: 'attached' }); }); test('run zq', async ({ page }) => { - await page.goto('/test/pages/zq.html'); await page.fill('[name=input]', '1 2 3'); await page.fill('[name=script]', 'this * 10'); await page.click('button'); diff --git a/packages/zed-wasm/test/cjs/parse.test.js b/packages/zed-wasm/test/cjs/parse.test.js index 8850a5968d..904716fe7d 100644 --- a/packages/zed-wasm/test/cjs/parse.test.js +++ b/packages/zed-wasm/test/cjs/parse.test.js @@ -1,7 +1,14 @@ -const { parse, zq } = require('@brimdata/zed-wasm'); +const { initZedWasm } = require('@brimdata/zed-wasm'); + +let wasm; +beforeAll(async () => { + wasm = await initZedWasm( + require.resolve('@brimdata/zed-wasm/dist/main.wasm') + ); +}); test('parse works in cjs env', async () => { - const resp = await parse('n := "james"'); + const resp = await wasm.parse('n := "james"'); expect(resp).toMatchInlineSnapshot(` { "ast": [ @@ -29,7 +36,7 @@ test('parse works in cjs env', async () => { }); test('zq works in cjs env', async () => { - const resp = await zq({ + const resp = await wasm.zq({ input: '1 2 3', program: 'this * 10', }); diff --git a/packages/zed-wasm/test/node/parser.test.js b/packages/zed-wasm/test/node/parser.test.js index d4bcc12154..ffc8684760 100644 --- a/packages/zed-wasm/test/node/parser.test.js +++ b/packages/zed-wasm/test/node/parser.test.js @@ -1,6 +1,14 @@ import test from 'node:test'; import assert from 'node:assert'; -import { parse } from '@brimdata/zed-wasm'; +import { initZedWasm } from '@brimdata/zed-wasm'; + +let parse; +test.before(async () => { + const wasm = await initZedWasm( + require.resolve('@brimdata/zed-wasm/dist/main.wasm') + ); + parse = wasm.parse; +}); test('parser', async () => { const resp = await parse('me := "james"'); diff --git a/packages/zed-wasm/test/node/zq.test.js b/packages/zed-wasm/test/node/zq.test.js index 9dac91268d..d29252fc03 100644 --- a/packages/zed-wasm/test/node/zq.test.js +++ b/packages/zed-wasm/test/node/zq.test.js @@ -1,6 +1,14 @@ import test from 'node:test'; import assert from 'node:assert'; -import { zq } from '@brimdata/zed-wasm'; +import { initZedWasm } from '@brimdata/zed-wasm'; + +let zq; +test.before(async () => { + const wasm = await initZedWasm( + require.resolve('@brimdata/zed-wasm/dist/main.wasm') + ); + zq = wasm.zq; +}); test('input is string', async (t) => { const input = '1 2 3'; diff --git a/packages/zed-wasm/test/pages/parse.html b/packages/zed-wasm/test/pages/parse.html index 117a65c631..00b2d54c41 100644 --- a/packages/zed-wasm/test/pages/parse.html +++ b/packages/zed-wasm/test/pages/parse.html @@ -11,7 +11,11 @@ } - + +