feat: Add JSON Lines Support to KV Store (#126)

* feat: Add JSON Lines Support

* refactor: Value Check

* fix: IsLines

build/config/substation.libsonnet

@@ -42,7 +42,7 @@
         settings: { file: null, column: null, delimiter: ',', header: null },
       },
       json_file: {
-        settings: { file: null },
+        settings: { file: null, is_lines: false },
       },
       memory: {
         settings: { capacity: 1024 },

internal/kv/json_file.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/brexhq/substation/config"
@@ -12,15 +13,16 @@ import (
 	"github.com/brexhq/substation/internal/json"
 )
 
-// errJSONFileInvalid is returned when the file contains invalid JSON.
-var errJSONFileInvalid = fmt.Errorf("invalid JSON")
-
 // kvJSONFile is a read-only key-value store that is derived from a file containing
 // an object and stored in memory.
 type kvJSONFile struct {
 	// File contains the location of the text file. This can be either a path on local
 	// disk, an HTTP(S) URL, or an AWS S3 URL.
-	File   string `json:"file"`
+	File string `json:"file"`
+	// IsLines indicates that the file is a JSON Lines file. The first non-null value
+	// is returned when a key is found.
+	IsLines bool `json:"is_lines"`
+
 	mu     *sync.Mutex
 	object []byte
 }
@@ -49,6 +51,21 @@ func (store *kvJSONFile) Get(ctx context.Context, key string) (interface{}, erro
 	store.mu.Lock()
 	defer store.mu.Unlock()
 
+	// JSON Lines files are queried as an array and the first non-null value is returned.
+	// See https://github.com/tidwall/gjson#json-lines for more information.
+	if store.IsLines && !strings.HasPrefix(key, "..#.") {
+		key = "..#." + key
+		res := json.Get(store.object, key)
+
+		for _, v := range res.Array() {
+			if v.Exists() {
+				return v.Value(), nil
+			}
+		}
+
+		return nil, nil
+	}
+
 	res := json.Get(store.object, key)
 	if json.Types[res.Type] == "Null" {
 		return nil, nil
@@ -96,10 +113,6 @@ func (store *kvJSONFile) Setup(ctx context.Context) error {
 		return fmt.Errorf("kv: json_file: %v", err)
 	}
 
-	if !json.Valid(buf) {
-		return fmt.Errorf("kv: json_file: %v", errJSONFileInvalid)
-	}
-
 	store.object = buf
 	return nil
 }