chore: Use Go-based implementation as embedded extension to Python SDK

.github/workflows/pr_integration_tests.yml

@@ -151,9 +151,7 @@ jobs:
       - name: Install pip-tools
         run: pip install pip-tools
       - name: Install dependencies
-        run: |
-          make compile-protos-go
-          make install-python-ci-dependencies
+        run: make install-python-ci-dependencies
       - name: Setup Redis Cluster
         run: |
           docker pull vishnunair/docker-redis-cluster:latest

.github/workflows/unit_tests.yml

@@ -45,9 +45,7 @@ jobs:
       - name: Install pip-tools
         run: pip install pip-tools
       - name: Install dependencies
-        run: |
-          make compile-protos-go
-          make install-python-ci-dependencies
+        run: make install-python-ci-dependencies
       - name: Test Python
         env:
           SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}

Makefile

@@ -35,7 +35,7 @@ build: protos build-java build-docker build-html
 
 # Python SDK
 
-install-python-ci-dependencies: install-go-proto-dependencies
+install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies
 	cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt
 	cd sdk/python && COMPILE_GO=true python setup.py develop
 
@@ -76,14 +76,9 @@ test-python-universal-local:
 test-python-universal:
 	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
 
-test-python-go-server:
-	go build -o ${ROOT_DIR}/sdk/python/feast/binaries/server github.com/feast-dev/feast/go/cmd/server
+test-python-go-server: compile-go-lib
 	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --goserver sdk/python/tests
 
-test-python-go-server-lifecycle:
-	go build -o ${ROOT_DIR}/sdk/python/feast/binaries/server github.com/feast-dev/feast/go/cmd/server
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --goserverlifecycle sdk/python/tests
-
 format-python:
 	# Sort
 	cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/
@@ -123,21 +118,28 @@ build-java:
 build-java-no-tests:
 	${MVN} --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true -DskipITs=true -Drevision=${REVISION} clean package
 
-# Go SDK
+# Go SDK & embedded
 
 install-go-proto-dependencies:
 	go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
 	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
 
+install-go-ci-dependencies:
+	# ToDo: currently gopy installation doesn't work w/o explicit go get in the next two lines
+	# ToDo: there should be a better way to install gopy
+	go get golang.org/x/tools/cmd/goimports
+	go get github.com/go-python/gopy
+	go install github.com/go-python/gopy
+
 install-protoc-dependencies:
 	pip install grpcio-tools==1.34.0
 
 compile-protos-go: install-go-proto-dependencies install-protoc-dependencies
 	cd sdk/python && python setup.py build_go_protos
 
-compile-go-feature-server: compile-protos-go
-	go mod tidy
-	go build -o ${ROOT_DIR}/sdk/python/feast/binaries/server github.com/feast-dev/feast/go/cmd/server
+compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies
+	python -m pip install pybindgen==0.22.0
+	cd sdk/python && python setup.py build_go_lib
 
 test-go: compile-protos-go
 	go test ./...

go.mod

@@ -3,7 +3,9 @@ module github.com/feast-dev/feast
 go 1.17
 
 require (
+	github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40
 	github.com/ghodss/yaml v1.0.0
+	github.com/go-python/gopy v0.4.0
 	github.com/go-redis/redis/v8 v8.11.4
 	github.com/golang/protobuf v1.5.2
 	github.com/google/uuid v1.1.2
@@ -17,12 +19,21 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/google/flatbuffers v2.0.0+incompatible // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.9 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
-	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
-	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a // indirect
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
+	golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
+
+replace github.com/go-python/gopy v0.4.0 => github.com/pyalex/gopy v0.4.1-0.20220315220227-146bcd99e9f5