PyPi Python3 support

.gitignore

@@ -4,3 +4,6 @@ _tmp
 sdk/python/build
 sdk/js/node_modules
 sdk/js/package-lock.json
+sdk/python/dist/
+sdk/python/libopenstorage_openstorage.egg-info/
+sdk/python/setup.py

Makefile

@@ -1,11 +1,18 @@
+ifndef APIVER
+APIVER=$(shell cat api.swagger.json | jq -r '.info.version')
+endif
 
 all: docker-proto
 
+clean:
+	$(MAKE) -C sdk clean
+
 docker-proto:
 	docker run \
 		--privileged \
 		-v $(shell pwd):/go/src/github.com/libopenstorage/openstorage \
 		-e "GOPATH=/go" \
+		-e "APIVER=$(APIVER)" \
 		-e "DOCKER_PROTO=yes" \
 		-e "PATH=/bin:/usr/bin:/usr/local/bin:/go/bin" \
 		quay.io/openstorage/osd-proto-clients \

sdk/Makefile

@@ -1,12 +1,18 @@
 SUBDIRS := golang \
-	dart \
 	python \
 	ruby \
 	js
 
+CLEANDIRS=$(SUBDIRS:%=clean-%)
+
 all: $(SUBDIRS)
 
+clean: $(CLEANDIRS)
+
 $(SUBDIRS):
 	$(MAKE) -C $@
 
-.PHONY: all $(SUBDIRS)
+$(CLEANDIRS):
+	$(MAKE) -C $(@:clean-%=%) clean
+
+.PHONY: all $(SUBDIRS) $(CLEANDIRS)

sdk/dart/Makefile

@@ -16,6 +16,8 @@ export PATH
 
 all: grpc
 
+clean:
+
 grpc:
 	-mkdir -p lib/src/generated > /dev/null
 	$(PROTOC) -I $(PROTOSRC_PATH) \

sdk/golang/Makefile

@@ -14,6 +14,8 @@ endif
 
 all: grpc
 
+clean:
+
 grpc:
 ifndef HAS_PROTOC_GEN_GO
 	$(error "Please install grpc rools for ruby: gem install grpc && gem install grpc-tools")

sdk/js/Makefile

@@ -3,6 +3,8 @@ JSPROTOC := ./node_modules/.bin/grpc_tools_node_protoc
 APIDIR := ../..
 all: grpc
 
+clean:
+
 grpc:
 ifndef HAS_NPM
 	$(error "Please install npm")

sdk/python/Makefile

@@ -1,37 +1,45 @@
-HAS_VIRTUALENV := $(shell command -v virtualenv 2> /dev/null)
 APIDIR := ../..
 VEDIR := "build"
+PYTHON := python3
+PIP := pip3
 all: grpc
 
-grpc:
-ifndef HAS_VIRTUALENV
-	$(error "Please install virtualenv for python: pip install virtualenv")
+ifndef APIVER
+	$(error Must provide an APIVER value)
 endif
-	virtualenv $(VEDIR)
+
+grpc:
+	$(PYTHON) -m venv $(VEDIR)
 	bash -c "source $(VEDIR)/bin/activate && \
-	pip install grpcio grpcio-tools && \
-	python -m grpc_tools.protoc \
+	$(PIP) install --upgrade pip grpcio grpcio-tools setuptools wheel twine && \
+	$(PYTHON) -m grpc_tools.protoc \
 		-I$(APIDIR) \
 		-I$(GOPATH)/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
 		--python_out=. \
 		--grpc_python_out=. \
 		$(APIDIR)/api.proto" && \
 		mv api_pb2*.py openstorage
 	bash -c "source $(VEDIR)/bin/activate && \
-	python -m grpc_tools.protoc \
+	$(PYTHON) -m grpc_tools.protoc \
 		-I$(APIDIR) \
 		-I$(GOPATH)/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
 		--python_out=. \
 		--grpc_python_out=. \
 		$(GOPATH)/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/google/api/annotations.proto"
 	bash -c "source $(VEDIR)/bin/activate && \
-	python -m grpc_tools.protoc \
+	$(PYTHON) -m grpc_tools.protoc \
 		-I$(APIDIR) \
 		-I$(GOPATH)/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
 		--python_out=. \
 		--grpc_python_out=. \
 		$(GOPATH)/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/google/api/http.proto"
 	sed -i -e "s#^import api_pb2 as api__pb2#from openstorage import api_pb2 as api__pb2#" openstorage/api_pb2_grpc.py
+	sed -e "s#%%APIVER%%#$(APIVER)#" setup.py.tmpl > setup.py
+	bash -c "source $(VEDIR)/bin/activate && \
+		$(PYTHON) setup.py sdist bdist_wheel"
+
+clean:
+	sudo rm -rf $(VEDIR) setup.py dist libopenstorage_openstorage.egg-info
 
-.PHONY: all grpc
+.PHONY: all grpc clean
 

sdk/python/README.md

@@ -1,32 +1,35 @@
 # Python gRPC Client Library
 
-## Build
+# Build
 
 Type `make`.
 
-Requires `virtualenv`. To install type: `pip install virtualenv`
+# Installing
 
-## Using
+Installing through pip is now supported:
 
-Copy `openstorage` and `google` directories to your project.
+```
+   python3 -m venv sdk
+   source sdk/bin/activate
+   pip3 install libopenstorage-openstorage
+```
+
+> NOTE: Python 2.7 is no longer supported. This is for Python3 only.
 
-## Example
+# Example
 
 ```python
 #
-# Install:
-#   virtualenv sdk
-#   source sdk/bin/activate
-#   pip install grpcio grpcio-tools
-#
 # More info: https://grpc.io/docs/quickstart/python.html
 #
 import grpc
 
 from openstorage import api_pb2
 from openstorage import api_pb2_grpc
+from openstorage import connector
 
-channel = grpc.insecure_channel('localhost:9100')
+c = connector.Connector(endpoint='localhost:9100')
+channel = c.connect()
 
 try:
     # Cluster connection
@@ -48,5 +51,8 @@ except grpc.RpcError as e:
     print('Failed: code={0} msg={1}'.format(e.code(), e.details()))(build)
 ```
 
+# PyPi
+
+See https://dzone.com/articles/executable-package-pip-install
 
 

sdk/python/app.py

@@ -0,0 +1,36 @@
+#
+# Install:
+#   pip3 install --user --upgrade libopenstorage-openstorage
+#
+import grpc
+
+from openstorage import api_pb2
+from openstorage import api_pb2_grpc
+from openstorage import connector
+
+c = connector.Connector(endpoint='localhost:9100')
+#c = connector.Connector(endpoint='localhost:9300', cafile='/tmp/ca.pem', token_secret_name='px-k8s-user',
+#        token_secret_namespace='kube-system')
+#c = connector.Connector(endpoint='localhost:9200', secure=True, cafile='/tmp/ca.pem')
+#c = connector.Connector()
+channel = c.connect()
+
+try:
+    # Cluster connection
+    clusters = api_pb2_grpc.OpenStorageClusterStub(channel)
+    ic_resp = clusters.InspectCurrent(api_pb2.SdkClusterInspectCurrentRequest())
+    print('Conntected to {0} with status {1}'.format(ic_resp.cluster.id, api_pb2.Status.Name(ic_resp.cluster.status)))
+
+    # Create a volume
+    volumes = api_pb2_grpc.OpenStorageVolumeStub(channel)
+    v_resp = volumes.Create(api_pb2.SdkVolumeCreateRequest(
+        name="myvol",
+        spec=api_pb2.VolumeSpec(
+            size=10*1024*1024*1024,
+            ha_level=3,
+        )
+    ))
+    print('Volume id is {0}'.format(v_resp.volume_id))
+except grpc.RpcError as e:
+    print('Failed: code={0} msg={1}'.format(e.code(), e.details()))
+

sdk/python/openstorage/connector.py

@@ -0,0 +1,147 @@
+# Copyright (c) 2020 Portworx
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import grpc
+import base64
+import os
+from kubernetes import client, config
+
+from openstorage import api_pb2
+from openstorage import api_pb2_grpc
+
+# Environment Variables
+ENDPOINT_ENV_KEY = 'OPENSTORAGE_SDK_ENDPOINT'
+SECURE_ENV_KEY = 'OPENSTORAGE_SDK_SECURE'
+TOKEN_ENV_KEY = 'OPENSTORAGE_SDK_TOKEN'
+CAFILE_ENV_KEY = 'OPENSTORAGE_SDK_CAFILE'
+SECRET_NAME_ENV_KEY = 'OPENSTORAGE_SDK_SECRET_NAME'
+SECRET_NAMESPACE_ENV_KEY = 'OPENSTORAGE_SDK_SECRET_NAMESPACE'
+
+
+class Connector(object):
+    """
+    Connects to OpenStorage SDK server.
+
+    Manages connection and setup of the gRPC when using tokens and TLS.
+    The token may be passed in or fetched from a Kubernetes secret.
+    """
+    def __init__(self, endpoint='', secure=False, token='', cafile='',
+                 token_secret_namespace='', token_secret_name=''):
+        """
+        :param endpoint: gRPC endpoint to OpenStorage SDK server
+        :type endpoint: str
+
+        :param secure: use TLS for the connection
+        :type secure: bool
+
+        :param token: OpenStorage Auth token
+        :type token: str
+
+        :param cafile: Path to CA file if required for TLS. If not provided
+            and 'secure' is enabled, then the CA must be part of the host.
+        :type cafile: str
+
+        :param token_secret_name: Name of the Kubernetes secret containing
+            the OpenStorage Auth token
+        :type token_secret_name: str
+
+        :param token_secret_namespace: Name of the namespace in Kubernetes
+            containing the secret object with the OpenStorage Auth token
+        :type token_secret_namespace: str
+        """
+        self.endpoint = endpoint
+        self.secure = secure
+        self.token = token
+        self.cafile = cafile
+        self.token_secret_name = token_secret_name
+        self.token_secret_namespace = token_secret_namespace
+
+        # Overrite settings using environment
+        self._from_environment()
+
+        if self.endpoint == '':
+            raise Exception('Endpoint not provided')
+
+        # Check if secret must be fetched from Kubernetes
+        if self._use_k8s_secret():
+            self.token = self._get_kubernetes_secret()
+
+
+    def connect(self, opts=None):
+        """
+        Connect to server
+
+        :param opts:gRPC channel options if any
+        :return: A gRPC channel
+        """
+        if self._is_secure():
+            return self._get_secure_channel(opts)
+        return grpc.insecure_channel(self.endpoint, opts)
+
+
+    def _from_environment(self):
+        e = os.getenv(ENDPOINT_ENV_KEY)
+        if e:
+            self.endpoint = e
+        e = os.getenv(SECURE_ENV_KEY)
+        if e:
+            self.secure = e.lower() in ['true', '1', 't', 'y', 'yes']
+        e = os.getenv(TOKEN_ENV_KEY)
+        if e:
+            self.token = e
+        e = os.getenv(CAFILE_ENV_KEY)
+        if e:
+            self.cafile = e
+        e = os.getenv(SECRET_NAME_ENV_KEY)
+        if e:
+            self.token_secret_name = e
+        e = os.getenv(SECRET_NAMESPACE_ENV_KEY)
+        if e:
+            self.token_secret_namespace = e
+
+
+    def _use_k8s_secret(self):
+        return self.token_secret_name != '' and self.token_secret_namespace != ''
+
+
+    def _is_secure(self):
+        return self.secure or self.cafile != ''
+
+
+    def _is_authenticated(self):
+        return self.token != ''
+
+
+    def _get_kubernetes_secret(self):
+        config.load_kube_config()
+        v1 = client.CoreV1Api()
+        ret = v1.read_namespaced_secret(self.token_secret_name, self.token_secret_namespace)
+        return str(base64.b64decode(ret.data['auth-token']), "utf-8")
+
+
+    def _get_secure_channel(self, opts=None):
+        # Setup CA if any
+        with open(self.cafile, 'rb') as f:
+            capem = f.read()
+        creds = grpc.ssl_channel_credentials(root_certificates=capem)
+
+        # Setup authentication if any
+        if self._is_authenticated():
+            auth = grpc.access_token_call_credentials(self.token)
+            return grpc.secure_channel(self.endpoint, grpc.composite_channel_credentials(creds, auth), opts)
+
+        return grpc.secure_channel(self.endpoint, creds, opts)
+
+