detect top-level JSON arrays in zio/anyio.Reader (#3124)

brimdata · Sep 29, 2021 · 5407d33 · 5407d33
1 parent cc83c8f
commit 5407d33
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 6 deletions.
diff --git a/service/ztests/load-garbage.yaml b/service/ztests/load-garbage.yaml
@@ -19,7 +19,7 @@ outputs:
       	zson: identifier "This" must be enum and requires decorator
       	zng: malformed zng record
       	csv: auto-detection not supported
-      	json: auto-detection not supported
+      	json: invalid character 'T' looking for beginning of value
       	parquet: auto-detection not supported
       	zst: auto-detection not supported
       status code 400: no records in request
diff --git a/zio/anyio/lookup.go b/zio/anyio/lookup.go
@@ -25,7 +25,7 @@ func lookupReader(r io.Reader, zctx *zson.Context, opts ReaderOpts) (zio.Reader,
 	case "zeek":
 		return zeekio.NewReader(r, zctx)
 	case "json":
-		return jsonio.NewReader(r, zctx)
+		return jsonio.NewReader(r, zctx), nil
 	case "zjson":
 		return zjsonio.NewReader(r, zctx), nil
 	case "zng":

diff --git a/zio/anyio/reader.go b/zio/anyio/reader.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/brimdata/zed/zio"
+	"github.com/brimdata/zed/zio/jsonio"
 	"github.com/brimdata/zed/zio/tzngio"
 	"github.com/brimdata/zed/zio/zeekio"
 	"github.com/brimdata/zed/zio/zjsonio"
@@ -57,6 +58,15 @@ func NewReaderWithOpts(r io.Reader, zctx *zson.Context, opts ReaderOpts) (zio.Re
 	}
 	track.Reset()
 
+	// JSON comes after ZSON because we want the ZSON reader to handle
+	// top-level JSON objects and the JSON reader to handle top-level
+	// JSON arrays.
+	jsonErr := match(jsonio.NewReader(track, zson.NewContext()), "json")
+	if jsonErr == nil {
+		return jsonio.NewReader(recorder, zctx), nil
+	}
+	track.Reset()
+
 	// For the matching reader, force validation to true so we are extra
 	// careful about auto-matching ZNG.  Then, once matched, relaxed
 	// validation to the user setting in the actual reader returned.
@@ -77,7 +87,6 @@ func NewReaderWithOpts(r io.Reader, zctx *zson.Context, opts ReaderOpts) (zio.Re
 	//track.Reset()
 
 	csvErr := errors.New("csv: auto-detection not supported")
-	jsonErr := errors.New("json: auto-detection not supported")
 	parquetErr := errors.New("parquet: auto-detection not supported")
 	zstErr := errors.New("zst: auto-detection not supported")
 	return nil, joinErrs([]error{tzngErr, zeekErr, zjsonErr, zsonErr, zngErr, csvErr, jsonErr, parquetErr, zstErr})

diff --git a/zio/anyio/ztests/fake-zng.yaml b/zio/anyio/ztests/fake-zng.yaml
@@ -16,6 +16,6 @@ outputs:
       	zson: zson syntax error
       	zng: zng type ID out of range
       	csv: auto-detection not supported
-      	json: auto-detection not supported
+      	json: invalid character '\x00' looking for beginning of value
       	parquet: auto-detection not supported
       	zst: auto-detection not supported
diff --git a/zio/anyio/ztests/json-array.yaml b/zio/anyio/ztests/json-array.yaml
@@ -0,0 +1,9 @@
+zed: '*'
+
+input: |
+  [{"a":1,"b":2},{"b":3,"a":4}]
+
+output: |
+  {a:1,b:2}
+  {a:4,b:3}
+
diff --git a/zio/jsonio/reader.go b/zio/jsonio/reader.go
@@ -14,7 +14,7 @@ type Reader struct {
 	decoder *json.Decoder
 }
 
-func NewReader(r io.Reader, zctx *zson.Context) (*Reader, error) {
+func NewReader(r io.Reader, zctx *zson.Context) *Reader {
 	d := json.NewDecoder(r)
 	// Prime d's buffer so we can check for an array.
 	d.More()
@@ -26,7 +26,7 @@ func NewReader(r io.Reader, zctx *zson.Context) (*Reader, error) {
 	return &Reader{
 		zctx:    zctx,
 		decoder: d,
-	}, nil
+	}
 }
 
 func (r *Reader) Read() (*zed.Record, error) {