Fix documentation building for Feast SDK (#1400)

* Update documentation and proto building

Signed-off-by: Willem Pienaar <git@willem.co>

* Automatically build API docs

Signed-off-by: Willem Pienaar <git@willem.co>

* Update formatting

Signed-off-by: Willem Pienaar <git@willem.co>

* Fix SDK release building

Signed-off-by: Willem Pienaar <git@willem.co>

* Remove explicit compilation step from proto building

Signed-off-by: Willem Pienaar <git@willem.co>

* Fix Proto CI for Go

Signed-off-by: Willem Pienaar <git@willem.co>

* Fix Proto CI for Python

Signed-off-by: Willem Pienaar <git@willem.co>

* Fix Python Integration Test Dependencies

Signed-off-by: Willem Pienaar <git@willem.co>

* Add printenv to compile protos

Signed-off-by: Willem Pienaar <git@willem.co>

* Add mypy-protobuf installation to compile-protos-python

Signed-off-by: Willem Pienaar <git@willem.co>

* Install Mypy

Signed-off-by: Willem Pienaar <git@willem.co>

* Fix telemetry test

Signed-off-by: Willem Pienaar <git@willem.co>

.github/workflows/integration_tests.yml

@@ -17,6 +17,10 @@ jobs:
           export_default_credentials: true
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           service_account_key: ${{ secrets.GCP_SA_KEY }}
+      - name: Install dependencies
+        run: make install-python-ci-dependencies
+      - name: Compile protos
+        run: make compile-protos-python
       - name: Install python
         run: make install-python
       - name: Test python

.github/workflows/pr_integration_tests.yml

@@ -28,6 +28,11 @@ jobs:
           export_default_credentials: true
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           service_account_key: ${{ secrets.GCP_SA_KEY }}
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: make install-python-ci-dependencies
+      - name: Compile protos
+        run: make compile-protos-python
       - name: Install python
         run: make install-python
       - name: Test python

.github/workflows/unit_tests.yml

@@ -8,9 +8,13 @@ jobs:
     container: gcr.io/kf-feast/feast-ci:latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install python
+      - name: Install dependencies
+        run: make install-python-ci-dependencies
+      - name: Compile protos
+        run: make compile-protos-python
+      - name: Install Python
         run: make install-python
-      - name: Test python
+      - name: Test Python
         run: make test-python
 
   unit-test-go:
@@ -19,6 +23,8 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Install dependencies
+        run: make install-go-ci-dependencies
+      - name: Compile protos
         run: make compile-protos-go
-      - name: Test go
+      - name: Test
         run: make test-go

.gitignore

@@ -178,14 +178,15 @@ dmypy.json
 # This pom should not be committed because it is only used during release / deployment.
 .flattened-pom.xml
 
-sdk/python/docs/html
-
-# Generated python code
-*_pb2.py
-*_pb2.pyi
-*_pb2_grpc.py
 
 # VSCode
 .bloop
 .metals
 *.code-workspace
+
+# Protos
+sdk/python/docs/source
+sdk/python/docs/html
+sdk/python/feast/protos/
+sdk/python/tensorflow_metadata
+sdk/go/protos/

.pre-commit-config.yaml

@@ -1,8 +1,10 @@
+default_stages:
+  - push
 repos:
   - repo: local
     hooks:
       - id: lint
         name: Lint
-        stages: [commit]
+        stages: [ push ]
         language: system
         entry: make lint

.readthedocs.yml

@@ -0,0 +1,14 @@
+version: 2
+
+sphinx:
+   configuration: sdk/python/docs/source/conf.py
+
+formats:
+   - pdf
+
+python:
+  version: 3.7
+  install:
+      - requirements: sdk/python/requirements-ci.txt
+      - path: sdk/python/
+        method: setuptools

Makefile

@@ -44,12 +44,14 @@ install-python-ci-dependencies:
 package-protos:
 	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
 
-compile-protos-python: install-python-ci-dependencies
+compile-protos-python:
+	printenv
+	pip install --ignore-installed mypy-protobuf || echo "Mypy could not be installed"
 	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;)
 	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;)
 	cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto
 
-install-python: compile-protos-python
+install-python:
 	python -m pip install -e sdk/python -U --use-deprecated=legacy-resolver
 
 test-python:
@@ -74,7 +76,7 @@ install-go-ci-dependencies:
 	go get -u github.com/golang/protobuf/protoc-gen-go
 	go get -u golang.org/x/lint/golint
 
-compile-protos-go: install-go-ci-dependencies
+compile-protos-go:
 	cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos/ tensorflow_metadata/proto/v0/*.proto
 	$(foreach dir,types serving core storage,cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos feast/$(dir)/*.proto;)
 
@@ -124,22 +126,9 @@ install-dependencies-proto-docs:
 	mv protoc3/include/* $$HOME/include
 
 compile-protos-docs:
-	cd ${ROOT_DIR}/protos; 
-	mkdir -p ../dist/grpc
-	protoc --docs_out=../dist/grpc feast/*/*.proto || \
-	cd ${ROOT_DIR}; $(MAKE) install-dependencies-proto-docs && 	cd ${ROOT_DIR}/protos; PATH=$$HOME/bin:$$PATH protoc -I $$HOME/include/ -I . --docs_out=../dist/grpc feast/*/*.proto
+	rm -rf 	$(ROOT_DIR)/dist/grpc
+	mkdir -p dist/grpc;
+	cd ${ROOT_DIR}/protos && protoc --docs_out=../dist/grpc feast/*/*.proto
 
-clean-html:
-	rm -rf 	$(ROOT_DIR)/dist
-
-build-html: clean-html
-	mkdir -p $(ROOT_DIR)/dist/python
-	mkdir -p $(ROOT_DIR)/dist/grpc
-
-	# Build Protobuf documentation
-	$(MAKE) compile-protos-docs
-
-	# Build Python SDK documentation
-	$(MAKE) compile-protos-python
-	cd 	$(ROOT_DIR)/sdk/python/docs && $(MAKE) html
-	cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python
+build-sphinx: compile-protos-python
+	cd 	$(ROOT_DIR)/sdk/python/docs && $(MAKE) build-api-source

sdk/python/.gitignore

@@ -115,5 +115,4 @@ dmypy.json
 .pyre/
 
 .vscode/*
-test.py
 playground

sdk/python/docs/Makefile

@@ -15,15 +15,17 @@ help:
 
 .PHONY: help Makefile source
 
-clean:
-	rm -rf $(PROJECT_ROOT)/sdk/python/docs/html
+clean-source:
 	rm -rf $(PROJECT_ROOT)/sdk/python/docs/source
 
-build-api-source:
+clean-html:
+	rm -rf $(PROJECT_ROOT)/sdk/python/docs/html
+
+build-api-source: clean-source
 	sphinx-apidoc -f -o source $(PROJECT_ROOT)/sdk/python/feast
 	cp conf.py index.rst source/
 
-html: clean build-api-source
+html: clean-html build-api-source
 	sphinx-build -b html source html
 	rm -rf $(PROJECT_ROOT)/sdk/python/docs/source
 

sdk/python/setup.py

@@ -11,12 +11,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import logging
 import os
 import re
 import subprocess
 
-from setuptools import find_packages, setup
+from setuptools import find_packages
+
+try:
+    from setuptools import setup
+    from setuptools.command.install import install
+    from setuptools.command.develop import develop
+    from setuptools.command.egg_info import egg_info
+    from setuptools.command.sdist import sdist
+
+except ImportError:
+    from distutils.core import setup
+    from distutils.command.install import install
 
 NAME = "feast"
 DESCRIPTION = "Python SDK for Feast"
@@ -68,6 +79,33 @@
     r"^(?:[\/\w-]+)?(?P<version>[vV]?\d+(?:\.\d+){0,2}[^\+]*)(?:\+.*)?$"
 )
 
+proto_dirs = [x for x in os.listdir(repo_root + "/protos/feast")]
+proto_dirs.remove("third_party")
+
+
+def pre_install_build():
+    subprocess.check_call("make compile-protos-python", shell=True, cwd=f"{repo_root}")
+    subprocess.check_call("make build-sphinx", shell=True, cwd=f"{repo_root}")
+
+
+class CustomInstallCommand(install):
+    def do_egg_install(self):
+        pre_install_build()
+        install.do_egg_install(self)
+
+
+class CustomDevelopCommand(develop):
+    def run(self):
+        pre_install_build()
+        develop.run(self)
+
+
+class CustomEggInfoCommand(egg_info):
+    def run(self):
+        pre_install_build()
+        egg_info.run(self)
+
+
 setup(
     name=NAME,
     author=AUTHOR,
@@ -76,13 +114,14 @@
     long_description_content_type="text/markdown",
     python_requires=REQUIRES_PYTHON,
     url=URL,
-    packages=find_packages(exclude=("tests",)),
+    packages=find_packages(exclude=("tests",)) + ['.'],
     install_requires=REQUIRED,
     # https://stackoverflow.com/questions/28509965/setuptools-development-requirements
     # Install dev requirements with: pip install -e .[dev]
     extras_require={
         "dev": ["mypy-protobuf==1.*", "grpcio-testing==1.*"],
         "validation": ["great_expectations==0.13.2", "pyspark==3.0.1"],
+        "docs": ["grpcio-tools"],
     },
     include_package_data=True,
     license="Apache",
@@ -96,12 +135,19 @@
     ],
     entry_points={"console_scripts": ["feast=feast.cli:cli"]},
     use_scm_version={"root": "../..", "relative_to": __file__, "tag_regex": TAG_REGEX},
-    setup_requires=["setuptools_scm"],
+    setup_requires=["setuptools_scm", "grpcio-tools", "mypy-protobuf", "sphinx"],
     package_data={
         "": [
             "protos/feast/**/*.proto",
             "protos/feast/third_party/grpc/health/v1/*.proto",
             "protos/tensorflow_metadata/proto/v0/*.proto",
+            "feast/protos/feast/**/*.py",
+            "tensorflow_metadata/proto/v0/*.py"
         ],
     },
+    cmdclass={
+        "install": CustomInstallCommand,
+        "develop": CustomDevelopCommand,
+        "egg_info": CustomEggInfoCommand,
+    },
 )