Brought back Proto definitions in project, use buf and replace Line…

…s to use `repeated string`

buf.gen.yaml

@@ -0,0 +1,9 @@
+version: v1
+managed:
+  # We are now using managed mode for now because I did not find how to support `;pbsubstreams` package
+  # which we use currently.
+  enabled: false
+plugins:
+  - name: go
+    out: pb
+    opt: paths=source_relative

buf.work.yaml

@@ -0,0 +1,3 @@
+version: v1
+directories:
+  - proto

bundler/writer/buffered_test.go

@@ -53,7 +53,8 @@ func TestNewBufferedIO(t *testing.T) {
 				require.NoError(t, err)
 				writtenFiles := listFiles(workingDir)
 
-				require.NoError(t, uploadeable.Upload(context.Background(), output))
+				_, err = uploadeable.Upload(context.Background(), output)
+				require.NoError(t, err)
 
 				assert.Len(t, writtenFiles, 0)
 				assert.Equal(t, map[string][]byte{
@@ -78,7 +79,8 @@ func TestNewBufferedIO(t *testing.T) {
 
 				writtenFiles := listFiles(workingDir)
 
-				require.NoError(t, uploadeable.Upload(context.Background(), output), "upload file")
+				_, err = uploadeable.Upload(context.Background(), output)
+				require.NoError(t, err, "upload file")
 
 				assert.ElementsMatch(t, []string{"/0000000000-0000000010.tmp.jsonl"}, writtenFiles)
 				assert.Equal(t, map[string][]byte{

encoder/lines.go

@@ -2,10 +2,12 @@ package encoder
 
 import (
 	"fmt"
+	"reflect"
+	"unsafe"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/streamingfast/substreams-sink-files/bundler/writer"
-	pnfilesink "github.com/streamingfast/substreams-sink-files/pb"
+	pbsinkfiles "github.com/streamingfast/substreams-sink-files/pb/substreams/sink/files/v1"
 	pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1"
 )
 
@@ -19,15 +21,22 @@ func NewLineEncoder() *LinesEncoder {
 func (l *LinesEncoder) EncodeTo(output *pbsubstreams.ModuleOutput, writer writer.Writer) error {
 	// FIXME: Improve by using a customized probably decoder, maybe Vitess, or we could
 	// even create our own which should be quiter simpler and could even reduce allocations
-	lines := &pnfilesink.Lines{}
+	lines := &pbsinkfiles.Lines{}
 	if err := proto.Unmarshal(output.GetMapOutput().Value, lines); err != nil {
 		return fmt.Errorf("failed to unmarhsall lines: %w", err)
 	}
 
 	for _, line := range lines.Lines {
-		writer.Write(line)
+		writer.Write(unsafeGetBytes(line))
 		writer.Write([]byte("\n"))
 	}
 
 	return nil
 }
+
+// unsafeGetBytes get the `[]byte` value out of a string without an allocation that `[]byte(s)` does.
+//
+// See https://stackoverflow.com/a/68195226/697930 and the post in general for background
+func unsafeGetBytes(s string) []byte {
+	return unsafe.Slice((*byte)(unsafe.Pointer((*reflect.StringHeader)(unsafe.Pointer(&s)).Data)), len(s))
+}

encoder/lines_test.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/streamingfast/bstream"
 	"github.com/streamingfast/substreams-sink-files/bundler/writer"
-	pbfilesink "github.com/streamingfast/substreams-sink-files/pb"
+	pbsinkfiles "github.com/streamingfast/substreams-sink-files/pb/substreams/sink/files/v1"
 	pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1"
 	"github.com/stretchr/testify/assert"
 	"github.com/test-go/testify/require"
@@ -16,30 +16,30 @@ import (
 func TestLinesEncoder_EncodeTo(t *testing.T) {
 	tests := []struct {
 		name      string
-		lines     *pbfilesink.Lines
+		lines     *pbsinkfiles.Lines
 		expected  []byte
 		assertion assert.ErrorAssertionFunc
 	}{
 		{
 			"no line",
-			&pbfilesink.Lines{Lines: nil},
+			&pbsinkfiles.Lines{Lines: nil},
 			nil,
 			assert.NoError,
 		},
 		{
 			"one line",
-			&pbfilesink.Lines{Lines: [][]byte{[]byte(`{"a":1}`)}},
-			[]byte(`{"a":1}`),
+			&pbsinkfiles.Lines{Lines: []string{`{"a":1}`}},
+			[]byte(`{"a":1}` + "\n"),
 			assert.NoError,
 		},
 		{
 			"three line",
-			&pbfilesink.Lines{Lines: [][]byte{
-				[]byte(`{"a":1}`),
-				[]byte(`{"b":2}`),
-				[]byte(`{"c":3}`),
+			&pbsinkfiles.Lines{Lines: []string{
+				`{"a":1}`,
+				`{"b":2}`,
+				`{"c":3}`,
 			}},
-			[]byte(`{"a":1}` + "\n" + `{"b":2}` + "\n" + `{"c":3}`),
+			[]byte(`{"a":1}` + "\n" + `{"b":2}` + "\n" + `{"c":3}` + "\n"),
 			assert.NoError,
 		},
 	}

pb/generate.sh

@@ -16,60 +16,65 @@
 ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
 
 # Protobuf definitions
-FILESINK_PROTO="$ROOT/../substreams-eth-token-transfers/proto"
+PROTO=${1:-"$ROOT/proto"}
 
 function main() {
   checks
 
   set -e
-  pushd "$ROOT/pb" > /dev/null
+  pushd "$ROOT" >/dev/null
 
-    generate "filesink.proto"
+  generate_proto
 
-    echo "generate.sh - `date` - `whoami`" > $ROOT/pb/last_generate.txt
-    echo "streamingfast/proto revision: `GIT_DIR=$ROOT/.git git rev-parse HEAD`" >> $ROOT/pb/last_generate.txt
-  popd > /dev/null
-}
+  popd >/dev/null
+
+  echo "generate.sh - `date` - `whoami`" > $ROOT/pb/last_generate.txt
+  echo "streamingfast/substreams-sink-files revision: `GIT_DIR=$ROOT/.git git rev-parse HEAD`" >> $ROOT/pb/last_generate.txt
 
-# usage
-# - generate <protoPath>
-# - generate <protoBasePath/> [<file.proto> ...]
-function generate() {
-    base=""
-    if [[ "$#" -gt 1 ]]; then
-      base="$1"; shift
-    fi
+  echo "Done"
+}
 
-    for file in "$@"; do
-      protoc -I$FILESINK_PROTO \
-        --go_out=. \
-        --go_opt=paths=source_relative \
-        --go-grpc_out=. \
-        --go_opt="Mfilesink.proto=github.com/streamingfast/substreams-sink-files/pb;pbfilesink" \
-        --go-grpc_opt=paths=source_relative,require_unimplemented_servers=false \
-         $base$file
-    done
+function generate_proto() {
+  echo "Generating Substreams Sink Files Protobuf bindings via 'buf'"
+  buf generate proto
 }
 
 function checks() {
+  result=`printf "" | buf --version 2>&1 | grep -Eo '1\.(1[0-9]+|[2-9][0-9]+)\.'`
+  if [[ "$result" == "" ]]; then
+    echo "The 'buf' binary is either missing or is not recent enough (at `which buf || echo N/A`)."
+    echo ""
+    echo "To fix your problem, on Mac OS, perform this command:"
+    echo ""
+    echo "  brew install bufbuild/buf/buf"
+    echo ""
+    echo "On other system, refers to https://docs.buf.build/installation"
+    echo ""
+    echo "If everything is working as expetcted, the command:"
+    echo ""
+    echo "  buf --version"
+    echo ""
+    echo "Should print '1.11.0' (or newer)"
+    exit 1
+  fi
+
   # The old `protoc-gen-go` did not accept any flags. Just using `protoc-gen-go --version` in this
   # version waits forever. So we pipe some wrong input to make it exit fast. This in the new version
   # which supports `--version` correctly print the version anyway and discard the standard input
   # so it's good with both version.
-  result=`printf "" | protoc-gen-go --version 2>&1 | grep -Eo v[0-9\.]+`
+  result=`printf "" | protoc-gen-go --version 2>&1 | grep -Eo 'v1.(2[7-9]|[3-9][0-9]+)\.'`
   if [[ "$result" == "" ]]; then
-    echo "Your version of 'protoc-gen-go' (at `which protoc-gen-go`) is not recent enough."
+    echo "Plugin 'protoc-gen-go' is either missing or is not recent enough (at `which protoc-gen-go || echo N/A`)."
     echo ""
-    echo "To fix your problem, perform those commands:"
+    echo "To fix your problem, perform this command:"
     echo ""
-    echo "  go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.25.0"
-    echo "  go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0"
+    echo "  go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.0"
     echo ""
     echo "If everything is working as expetcted, the command:"
     echo ""
     echo "  protoc-gen-go --version"
     echo ""
-    echo "Should print 'protoc-gen-go v1.25.0' (if it just hangs, you don't have the correct version)"
+    echo "Should print 'protoc-gen-go v1.27.0' (if it just hangs, you don't have the correct version)"
     exit 1
   fi
 }

pb/last_generate.txt

@@ -1,2 +1,2 @@
-generate.sh - Fri  2 Dec 2022 10:03:40 EST - julien
-streamingfast/proto revision: c1bdc56dfebd326f27ef2c4fc9f3387285872b47
+generate.sh - Fri Jan  6 14:05:20 EST 2023 - maoueh
+streamingfast/substreams-sink-files revision: cc76c8b3a7ab3e8053006a2831e8d27b7a1f82bb