add support for VNG in API responses and load requests (#4345)

* Send a VNG response for an API request whose negotiated content type
  is application/x-vng.

* When loading data via POST /pool/{pool}/branch/{branch}, if the
  request content type is application/x-vng, copy the request body to a
  temporary file for the benefit of the VNG reader, which needs a reader
  that implements io.ReaderAt and io.Seeker.

Closes #4251.

api/mime.go

@@ -15,6 +15,7 @@ const (
 	MediaTypeLine        = "application/x-line"
 	MediaTypeNDJSON      = "application/x-ndjson"
 	MediaTypeParquet     = "application/x-parquet"
+	MediaTypeVNG         = "application/x-vng"
 	MediaTypeZeek        = "application/x-zeek"
 	MediaTypeZJSON       = "application/x-zjson"
 	MediaTypeZNG         = "application/x-zng"
@@ -54,6 +55,8 @@ func MediaTypeToFormat(s string, dflt string) (string, error) {
 		return "ndjson", nil
 	case MediaTypeParquet:
 		return "parquet", nil
+	case MediaTypeVNG:
+		return "vng", nil
 	case MediaTypeZeek:
 		return "zeek", nil
 	case MediaTypeZJSON:
@@ -80,6 +83,8 @@ func FormatToMediaType(format string) string {
 		return MediaTypeNDJSON
 	case "parquet":
 		return MediaTypeParquet
+	case "vng":
+		return MediaTypeVNG
 	case "zeek":
 		return MediaTypeZeek
 	case "zjson":

api/queryio/writer.go

@@ -10,6 +10,7 @@ import (
 	"github.com/brimdata/zed/zio"
 	"github.com/brimdata/zed/zio/anyio"
 	"github.com/brimdata/zed/zio/jsonio"
+	"github.com/brimdata/zed/zio/vngio"
 )
 
 type controlWriter interface {
@@ -44,7 +45,13 @@ func NewWriter(w io.WriteCloser, format string, flusher http.Flusher, ctrl bool)
 	case "ndjson":
 		d.writer = jsonio.NewWriter(w)
 	default:
-		d.writer, err = anyio.NewWriter(zio.NopCloser(w), anyio.WriterOpts{Format: format})
+		d.writer, err = anyio.NewWriter(zio.NopCloser(w), anyio.WriterOpts{
+			Format: format,
+			VNG: vngio.WriterOpts{
+				ColumnThresh: vngio.DefaultColumnThresh,
+				SkewThresh:   vngio.DefaultSkewThresh,
+			},
+		})
 	}
 	return d, err
 }

service/handlers.go

@@ -360,8 +360,8 @@ func handleBranchLoad(c *Core, w *ResponseWriter, r *Request) {
 		w.Error(err)
 		return
 	}
-	if format == "parquet" {
-		// This format requires a reader that implements io.ReaderAt and
+	if format == "parquet" || format == "vng" {
+		// These formats require a reader that implements io.ReaderAt and
 		// io.Seeker.  Copy the reader to a temporary file and use that.
 		//
 		// TODO: Add a way to disable this or limit file size.

service/ztests/curl-load-vng.yaml

@@ -0,0 +1,21 @@
+script: |
+  source service.sh
+  zed create -q test
+  zq -f vng in.zson |
+    curl -H Content-Type:application/x-vng --data-binary @- \
+    --fail $ZED_LAKE/pool/test/branch/main | zq -z commit:=0 -
+  echo //
+  zed query -z 'from test'
+
+inputs:
+  - name: in.zson
+    data: |
+      {x:1}
+  - name: service.sh
+
+outputs:
+  - name: stdout
+    data: |
+      {commit:0,warnings:[]([string])}
+      //
+      {x:1}

service/ztests/curl-query.yaml

@@ -10,9 +10,11 @@ script: |
   echo === application/vnd.apache.arrow.stream ===
   curl -H 'Accept: application/vnd.apache.arrow.stream' -d '{"query":"from test"}' $ZED_LAKE/query |
     zq -z -i arrows -
-  echo === application/x-parquet ===
-  curl -H 'Accept: application/x-parquet' -d '{"query":"from test"}' -o out.parquet $ZED_LAKE/query
-  zq -z out.parquet
+  for format in parquet vng; do
+    echo === application/x-$format ===
+    curl -H "Accept: application/x-$format" -d '{"query":"from test"}' -o out.$format $ZED_LAKE/query
+    zq -z out.$format
+  done
 
 inputs:
   - name: service.sh
@@ -60,3 +62,6 @@ outputs:
       === application/x-parquet ===
       {a:"hello",b:{c:"world",d:"goodbye"}}
       {a:"one",b:{c:"two",d:"three"}}
+      === application/x-vng ===
+      {a:"hello",b:{c:"world",d:"goodbye"}}
+      {a:"one",b:{c:"two",d:"three"}}