diff --git a/api/mime.go b/api/mime.go index 525e8ad3d0..059008d388 100644 --- a/api/mime.go +++ b/api/mime.go @@ -15,6 +15,7 @@ const ( MediaTypeLine = "application/x-line" MediaTypeNDJSON = "application/x-ndjson" MediaTypeParquet = "application/x-parquet" + MediaTypeVNG = "application/x-vng" MediaTypeZeek = "application/x-zeek" MediaTypeZJSON = "application/x-zjson" MediaTypeZNG = "application/x-zng" @@ -54,6 +55,8 @@ func MediaTypeToFormat(s string, dflt string) (string, error) { return "ndjson", nil case MediaTypeParquet: return "parquet", nil + case MediaTypeVNG: + return "vng", nil case MediaTypeZeek: return "zeek", nil case MediaTypeZJSON: @@ -80,6 +83,8 @@ func FormatToMediaType(format string) string { return MediaTypeNDJSON case "parquet": return MediaTypeParquet + case "vng": + return MediaTypeVNG case "zeek": return MediaTypeZeek case "zjson": diff --git a/api/queryio/writer.go b/api/queryio/writer.go index 0fa028c368..ce563f272e 100644 --- a/api/queryio/writer.go +++ b/api/queryio/writer.go @@ -10,6 +10,7 @@ import ( "github.com/brimdata/zed/zio" "github.com/brimdata/zed/zio/anyio" "github.com/brimdata/zed/zio/jsonio" + "github.com/brimdata/zed/zio/vngio" ) type controlWriter interface { @@ -44,7 +45,13 @@ func NewWriter(w io.WriteCloser, format string, flusher http.Flusher, ctrl bool) case "ndjson": d.writer = jsonio.NewWriter(w) default: - d.writer, err = anyio.NewWriter(zio.NopCloser(w), anyio.WriterOpts{Format: format}) + d.writer, err = anyio.NewWriter(zio.NopCloser(w), anyio.WriterOpts{ + Format: format, + VNG: vngio.WriterOpts{ + ColumnThresh: vngio.DefaultColumnThresh, + SkewThresh: vngio.DefaultSkewThresh, + }, + }) } return d, err } diff --git a/service/handlers.go b/service/handlers.go index b2abbeb607..f87d6afff6 100644 --- a/service/handlers.go +++ b/service/handlers.go @@ -360,8 +360,8 @@ func handleBranchLoad(c *Core, w *ResponseWriter, r *Request) { w.Error(err) return } - if format == "parquet" { - // This format requires a reader that implements io.ReaderAt and + if format == "parquet" || format == "vng" { + // These formats require a reader that implements io.ReaderAt and // io.Seeker. Copy the reader to a temporary file and use that. // // TODO: Add a way to disable this or limit file size. diff --git a/service/ztests/curl-load-vng.yaml b/service/ztests/curl-load-vng.yaml new file mode 100644 index 0000000000..ec2a397400 --- /dev/null +++ b/service/ztests/curl-load-vng.yaml @@ -0,0 +1,21 @@ +script: | + source service.sh + zed create -q test + zq -f vng in.zson | + curl -H Content-Type:application/x-vng --data-binary @- \ + --fail $ZED_LAKE/pool/test/branch/main | zq -z commit:=0 - + echo // + zed query -z 'from test' + +inputs: + - name: in.zson + data: | + {x:1} + - name: service.sh + +outputs: + - name: stdout + data: | + {commit:0,warnings:[]([string])} + // + {x:1} diff --git a/service/ztests/curl-query.yaml b/service/ztests/curl-query.yaml index 7f4f755187..e61c1e36e9 100644 --- a/service/ztests/curl-query.yaml +++ b/service/ztests/curl-query.yaml @@ -10,9 +10,11 @@ script: | echo === application/vnd.apache.arrow.stream === curl -H 'Accept: application/vnd.apache.arrow.stream' -d '{"query":"from test"}' $ZED_LAKE/query | zq -z -i arrows - - echo === application/x-parquet === - curl -H 'Accept: application/x-parquet' -d '{"query":"from test"}' -o out.parquet $ZED_LAKE/query - zq -z out.parquet + for format in parquet vng; do + echo === application/x-$format === + curl -H "Accept: application/x-$format" -d '{"query":"from test"}' -o out.$format $ZED_LAKE/query + zq -z out.$format + done inputs: - name: service.sh @@ -60,3 +62,6 @@ outputs: === application/x-parquet === {a:"hello",b:{c:"world",d:"goodbye"}} {a:"one",b:{c:"two",d:"three"}} + === application/x-vng === + {a:"hello",b:{c:"world",d:"goodbye"}} + {a:"one",b:{c:"two",d:"three"}}