bazel: Make use of new incremental pip installer

Signed-off-by: Ryan Northey <ryan@synca.io>
envoyproxy · Sep 9, 2021 · 5ee6d61 · 5ee6d61
1 parent de5bf9b
commit 5ee6d61
Show file tree

Hide file tree

Showing 30 changed files with 765 additions and 1,139 deletions.
diff --git a/.github/actions/pr_notifier/requirements.in b/.github/actions/pr_notifier/requirements.in
@@ -0,0 +1,2 @@
+pygithub
+slack_sdk
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -7,12 +7,12 @@ updates:
     interval: "daily"
 
 - package-ecosystem: "pip"
-  directory: "/test/extensions/filters/network/thrift_proxy"
+  directory: "/configs"
   schedule:
     interval: "daily"
 
 - package-ecosystem: "pip"
-  directory: "/contrib/kafka/filters/network/source"
+  directory: "/test/extensions/filters/network/thrift_proxy"
   schedule:
     interval: "daily"
 
@@ -26,48 +26,18 @@ updates:
   schedule:
     interval: "daily"
 
-- package-ecosystem: "pip"
-  directory: "/tools/docs"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/git"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/config_validation"
-  schedule:
-    interval: "daily"
-
 - package-ecosystem: "pip"
   directory: "/tools/dependency"
   schedule:
     interval: "daily"
 
-- package-ecosystem: "pip"
-  directory: "/tools/deprecate_version"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/distribution"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/protodoc"
-  schedule:
-    interval: "daily"
-
 - package-ecosystem: "pip"
   directory: "/tools/deprecate_features"
   schedule:
     interval: "daily"
 
 - package-ecosystem: "pip"
-  directory: "/tools/code_format"
+  directory: "/tools/deprecate_version"
   schedule:
     interval: "daily"
 
@@ -76,16 +46,6 @@ updates:
   schedule:
     interval: "daily"
 
-- package-ecosystem: "pip"
-  directory: "/configs"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/testing"
-  schedule:
-    interval: "daily"
-
 - package-ecosystem: "docker"
   directory: "/ci"
   schedule:

diff --git a/WORKSPACE b/WORKSPACE
@@ -16,6 +16,10 @@ load("//bazel:repositories_extra.bzl", "envoy_dependencies_extra")
 
 envoy_dependencies_extra()
 
+load("@base_pip3//:requirements.bzl", "install_deps")
+
+install_deps()
+
 load("//bazel:dependency_imports.bzl", "envoy_dependency_imports")
 
 envoy_dependency_imports()
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -637,14 +637,8 @@ def _com_google_absl():
     )
 
 def _com_google_protobuf():
-    # TODO(phlax): remove patch
-    #    patch is applied to update setuptools to version (0.5.4),
-    #    and can be removed once this has been updated in rules_python
-    #    see https://github.com/envoyproxy/envoy/pull/15236#issuecomment-788650946 for discussion
     external_http_archive(
         name = "rules_python",
-        patches = ["@envoy//bazel:rules_python.patch"],
-        patch_args = ["-p1"],
     )
 
     external_http_archive(

diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl
@@ -1,138 +1,28 @@
-load("@rules_python//python:pip.bzl", "pip_install")
+load("@rules_python//python:pip.bzl", "pip_install", "pip_parse")
 load("@proxy_wasm_cpp_host//bazel/cargo:crates.bzl", "proxy_wasm_cpp_host_fetch_remote_crates")
 load("//bazel/external/cargo:crates.bzl", "raze_fetch_remote_crates")
 
 # Python dependencies.
 def _python_deps():
-    pip_install(
+    pip_parse(
         name = "base_pip3",
-        requirements = "@envoy//tools/base:requirements.txt",
+        requirements_lock = "@envoy//tools/base:requirements.txt",
         extra_pip_args = ["--require-hashes"],
     )
-    pip_install(
-        name = "config_validation_pip3",
-        requirements = "@envoy//tools/config_validation:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-
-        # project_name = "PyYAML",
-        # project_url = "https://github.com/yaml/pyyaml",
-        # version = "5.4.1",
-        # release_date = "2021-01-20"
-        # use_category = ["devtools"],
-        # cpe = "cpe:2.3:a:pyyaml:pyyaml:*",
-    )
     pip_install(
         name = "configs_pip3",
         requirements = "@envoy//configs:requirements.txt",
         extra_pip_args = ["--require-hashes"],
-
-        # project_name = "Jinja",
-        # project_url = "http://palletsprojects.com/p/jinja",
-        # version = "2.11.2",
-        # release_date = "2020-04-13"
-        # use_category = ["test"],
-        # cpe = "cpe:2.3:a:palletsprojects:jinja:*",
-
-        # project_name = "MarkupSafe",
-        # project_url = "https://markupsafe.palletsprojects.com/en/1.1.x/",
-        # version = "1.1.1",
-        # release_date = "2019-02-23"
-        # use_category = ["test"],
-    )
-    pip_install(
-        name = "docs_pip3",
-        requirements = "@envoy//tools/docs:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-    )
-    pip_install(
-        name = "deps_pip3",
-        requirements = "@envoy//tools/dependency:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-    )
-    pip_install(
-        name = "distribution_pip3",
-        requirements = "@envoy//tools/distribution:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-    )
-    pip_install(
-        name = "git_pip3",
-        requirements = "@envoy//tools/git:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-    )
-    pip_install(
-        name = "kafka_pip3",
-        requirements = "@envoy//contrib/kafka/filters/network/source:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-
-        # project_name = "Jinja",
-        # project_url = "http://palletsprojects.com/p/jinja",
-        # version = "2.11.2",
-        # release_date = "2020-04-13"
-        # use_category = ["test"],
-        # cpe = "cpe:2.3:a:palletsprojects:jinja:*",
-
-        # project_name = "MarkupSafe",
-        # project_url = "https://markupsafe.palletsprojects.com/en/1.1.x/",
-        # version = "1.1.1",
-        # release_date = "2019-02-23"
-        # use_category = ["test"],
-    )
-    pip_install(
-        name = "protodoc_pip3",
-        requirements = "@envoy//tools/protodoc:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-
-        # project_name = "PyYAML",
-        # project_url = "https://github.com/yaml/pyyaml",
-        # version = "5.4.1",
-        # release_date = "2021-01-20"
-        # use_category = ["docs"],
-        # cpe = "cpe:2.3:a:pyyaml:pyyaml:*",
-    )
-    pip_install(
-        name = "pylint_pip3",
-        requirements = "@envoy//tools/code_format:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-    )
-    pip_install(
-        name = "testing_pip3",
-        requirements = "@envoy//tools/testing:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
     )
     pip_install(
         name = "thrift_pip3",
         requirements = "@envoy//test/extensions/filters/network/thrift_proxy:requirements.txt",
         extra_pip_args = ["--require-hashes"],
-
-        # project_name = "Apache Thrift",
-        # project_url = "http://thrift.apache.org/",
-        # version = "0.11.0",
-        # release_date = "2017-12-07"
-        # use_category = ["test"],
-        # cpe = "cpe:2.3:a:apache:thrift:*",
-
-        # project_name = "Six: Python 2 and 3 Compatibility Library",
-        # project_url = "https://six.readthedocs.io/",
-        # version = "1.15.0",
-        # release_date = "2020-05-21"
-        # use_category = ["test"],
     )
     pip_install(
         name = "fuzzing_pip3",
         requirements = "@rules_fuzzing//fuzzing:requirements.txt",
         extra_pip_args = ["--require-hashes"],
-
-        # project_name = "Abseil Python Common Libraries",
-        # project_url = "https://github.com/abseil/abseil-py",
-        # version = "0.11.0",
-        # release_date = "2020-10-27",
-        # use_category = ["test"],
-
-        # project_name = "Six: Python 2 and 3 Compatibility Library",
-        # project_url = "https://six.readthedocs.io/",
-        # version = "1.15.0",
-        # release_date = "2020-05-21"
-        # use_category = ["test"],
     )
 
 # Envoy deps that rely on a first stage of dependency loading in envoy_dependencies().

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
@@ -675,10 +675,11 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "Python rules for Bazel",
         project_desc = "Bazel rules for the Python language",
         project_url = "https://github.com/bazelbuild/rules_python",
-        version = "0.3.0",
-        sha256 = "934c9ceb552e84577b0faf1e5a2f0450314985b4d8712b2b70717dc679fdc01b",
-        release_date = "2021-06-23",
-        urls = ["https://github.com/bazelbuild/rules_python/releases/download/{version}/rules_python-{version}.tar.gz"],
+        version = "9f597623ccfbe430b0d81c82498e33b80b7aec88",
+        sha256 = "8d61fed6974f1e69e09243ca78c9ecf82f50fa3de64bb5df6b0b9061f9c9639b",
+        release_date = "2021-09-07",
+        strip_prefix = "rules_python-{version}",
+        urls = ["https://github.com/bazelbuild/rules_python/archive/{version}.tar.gz"],
         use_category = ["build"],
     ),
     rules_pkg = dict(

diff --git a/bazel/rules_python.patch b/bazel/rules_python.patch
diff --git a/ci/flaky_test/requirements.in b/ci/flaky_test/requirements.in
@@ -0,0 +1 @@
+slackclient
diff --git a/configs/BUILD b/configs/BUILD
@@ -20,8 +20,7 @@ py_binary(
         "*.yaml",
     ]),
     deps = [
-        requirement("Jinja2"),
-        requirement("MarkupSafe"),
+        requirement("jinja2"),
     ],
 )
 

diff --git a/contrib/kafka/filters/network/source/BUILD b/contrib/kafka/filters/network/source/BUILD
@@ -5,7 +5,7 @@ load(
     "envoy_contrib_package",
 )
 load("@rules_python//python:defs.bzl", "py_binary", "py_library")
-load("@kafka_pip3//:requirements.bzl", "requirement")
+load("@base_pip3//:requirements.bzl", "requirement")
 
 licenses(["notice"])  # Apache 2
 

diff --git a/contrib/kafka/filters/network/source/requirements.txt b/contrib/kafka/filters/network/source/requirements.txt
diff --git a/contrib/kafka/filters/network/test/BUILD b/contrib/kafka/filters/network/test/BUILD
@@ -5,7 +5,7 @@ load(
     "envoy_contrib_package",
 )
 load("@rules_python//python:defs.bzl", "py_binary")
-load("@kafka_pip3//:requirements.bzl", "requirement")
+load("@base_pip3//:requirements.bzl", "requirement")
 
 licenses(["notice"])  # Apache 2
 

diff --git a/contrib/kafka/filters/network/test/broker/integration_test/BUILD b/contrib/kafka/filters/network/test/broker/integration_test/BUILD
@@ -3,7 +3,7 @@ load(
     "envoy_contrib_package",
     "envoy_py_test",
 )
-load("@kafka_pip3//:requirements.bzl", "requirement")
+load("@base_pip3//:requirements.bzl", "requirement")
 
 licenses(["notice"])  # Apache 2
 

diff --git a/test/extensions/filters/network/thrift_proxy/requirements.in b/test/extensions/filters/network/thrift_proxy/requirements.in
@@ -0,0 +1,2 @@
+six
+thrift
diff --git a/tools/base/requirements.in b/tools/base/requirements.in
@@ -0,0 +1,30 @@
+aio.functional
+colorama
+coloredlogs
+coverage
+envoy.base.utils
+envoy.distribution.release
+envoy.distribution.verify
+envoy.gpg.sign
+flake8
+frozendict
+gitpython
+jinja2
+pep8-naming
+pygithub
+pyreadline
+pytest
+pytest-asyncio
+pytest-cov
+pytest-patches
+pyyaml
+setuptools
+sphinx
+sphinxcontrib-httpdomain
+sphinxcontrib-serializinghtml
+sphinx-copybutton
+sphinxext-rediraffe
+sphinx-rtd-theme
+sphinx-tabs
+verboselogs
+yapf