feat: avoid creating unused ThreadPools (#91)

* feat: avoid creating unused ThreadPools * docs: add requested TODOs
kubernetes-client · Dec 17, 2018 · 9d14077 · 9d14077
1 parent 9cf6542
commit 9d14077
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 0 deletions.
diff --git a/openapi/python-api_client.py.patch b/openapi/python-api_client.py.patch
@@ -0,0 +1,42 @@
+--- a/kubernetes/client/api_client.py
++++ b/kubernetes/client/api_client.py
+@@ -58,13 +58,15 @@ class ApiClient(object):
+         'datetime': datetime,
+         'object': object,
+     }
++    _pool = None
+
+-    def __init__(self, configuration=None, header_name=None, header_value=None, cookie=None):
++    def __init__(self, configuration=None, header_name=None, header_value=None,
++                 cookie=None, pool_threads=None):
+         if configuration is None:
+             configuration = Configuration()
+         self.configuration = configuration
++        self.pool_threads = pool_threads
+
+-        self.pool = ThreadPool()
+         self.rest_client = RESTClientObject(configuration)
+         self.default_headers = {}
+         if header_name is not None:
+@@ -74,8 +76,19 @@ class ApiClient(object):
+         self.user_agent = 'Swagger-Codegen/8.0.0-snapshot/python'
+
+     def __del__(self):
+-        self.pool.close()
+-        self.pool.join()
++        if self._pool:
++            self._pool.close()
++            self._pool.join()
++            self._pool = None
++
++    @property
++    def pool(self):
++        """Create thread pool on first request
++         avoids instantiating unused threadpool for blocking clients.
++        """
++        if self._pool is None:
++            self._pool = ThreadPool(self.pool_threads)
++        return self._pool
+
+     @property
+     def user_agent(self):
diff --git a/openapi/python-asyncio-api_client.py.patch b/openapi/python-asyncio-api_client.py.patch
@@ -0,0 +1,27 @@
+61a62
+>     _pool = None
+64c65
+<                  cookie=None):
+---
+>                  cookie=None, pool_threads=None):
+67a69
+>         self.pool_threads = pool_threads
+69d70
+<         self.pool = ThreadPool()
+79,80c80,92
+<         self.pool.close()
+<         self.pool.join()
+---
+>         if self._pool:
+>             self._pool.close()
+>             self._pool.join()
+>             self._pool = None
+> 
+>     @property
+>     def pool(self):
+>         """Create thread pool on first request
+>          avoids instantiating unused threadpool for blocking clients.
+>         """
+>         if self._pool is None:
+>             self._pool = ThreadPool(self.pool_threads)
+>         return self._pool
diff --git a/openapi/python-asyncio.sh b/openapi/python-asyncio.sh
@@ -66,22 +66,30 @@ if [ ${PACKAGE_NAME} == "client" ]; then
   find "${OUTPUT_DIR}" -path "${OUTPUT_DIR}/base" -prune -o -type f -a -name \*.md -exec sed -i 's/kubernetes_asyncio.client-python/client-python/g' {} +
 
   # workaround https://github.com/swagger-api/swagger-codegen/pull/7905
+  # TODO: Remove this when above merges
   find "${OUTPUT_DIR}/client" -type f -name \*.py ! -name '__init__.py' -exec sed -i '/^from .*models.*/d' {} \;
 
   # workaround https://github.com/swagger-api/swagger-codegen/pull/8204
   # + closing session
   # + support application/strategic-merge-patch+json
   # workaround https://github.com/swagger-api/swagger-codegen/pull/8797
   # + aiohttp without verify_ssl
+  # TODO: Remove this when above merges
   patch "${OUTPUT_DIR}/client/rest.py" "${SCRIPT_ROOT}/python-asyncio-rest.py.patch"
 
   # workaround https://github.com/swagger-api/swagger-codegen/pull/8401
+  # TODO: Remove this when above merges
+
   find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i 's/async=/async_req=/g' {} +
   find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i 's/async bool/async_req bool/g' {} +
   find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i "s/'async'/'async_req'/g" {} +
   find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i "s/async parameter/async_req parameter/g" {} +
   find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i "s/if not async/if not async_req/g" {} +
 
+  # workaround https://github.com/swagger-api/swagger-codegen/pull/8061 (thread pool only on demand)
+  # TODO: Remove this when above merges
+  patch "${OUTPUT_DIR}/client/api_client.py" "${SCRIPT_ROOT}/python-asyncio-api_client.py.patch"
+
   # fix imports
   find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i 's/import client\./import kubernetes_asyncio.client./g' {} +
   find "${OUTPUT_DIR}/client/" -type f -name \*.py -exec sed -i 's/from client/from kubernetes_asyncio.client/g' {} +

diff --git a/openapi/python.sh b/openapi/python.sh
@@ -54,12 +54,17 @@ kubeclient::generator::generate_client "${OUTPUT_DIR}"
 echo "--- Patching generated code..."
 
 # workaround https://github.com/swagger-api/swagger-codegen/pull/8401
+# TODO: Remove this when above merges
 find "${OUTPUT_DIR}/${PACKAGE_NAME}/" -type f -name \*.py -exec sed -i 's/async=/async_req=/g' {} +
 find "${OUTPUT_DIR}/${PACKAGE_NAME}/" -type f -name \*.py -exec sed -i 's/async bool/async_req bool/g' {} +
 find "${OUTPUT_DIR}/${PACKAGE_NAME}/" -type f -name \*.py -exec sed -i "s/'async'/'async_req'/g" {} +
 sed -i "s/if not async/if not async_req/g" "${OUTPUT_DIR}/${PACKAGE_NAME}/api_client.py"
 #
 
+# workaround https://github.com/swagger-api/swagger-codegen/pull/8061 (thread pool only on demand)
+# TODO: Remove this when above merges
+patch "${OUTPUT_DIR}/${PACKAGE_NAME}/api_client.py" "${SCRIPT_ROOT}/python-api_client.py.patch"
+
 find "${OUTPUT_DIR}/test" -type f -name \*.py -exec sed -i 's/\bclient/kubernetes.client/g' {} +
 find "${OUTPUT_DIR}" -path "${OUTPUT_DIR}/base" -prune -o -type f -a -name \*.md -exec sed -i 's/\bclient/kubernetes.client/g' {} +
 find "${OUTPUT_DIR}" -path "${OUTPUT_DIR}/base" -prune -o -type f -a -name \*.md -exec sed -i 's/kubernetes.client-python/client-python/g' {} +