Add support for --no-pre-install-wheels and --max-install-jobs.

pex/bin/pex.py

@@ -164,6 +164,41 @@ def configure_clp_pex_options(parser):
             "mode as well and support `--seed`ing."
         ),
     )
+    group.add_argument(
+        "--pre-install-wheels",
+        "--no-pre-install-wheels",
+        dest="pre_install_wheels",
+        default=True,
+        action=HandleBoolAction,
+        help=(
+            "Whether to pre-install third party dependency wheels. Pre-installed wheels will "
+            "always yield slightly faster PEX cold boot times; so they are used by default, but "
+            "they also slow down PEX build time. As the size of dependencies grows you may find a "
+            "tipping point where it makes sense to not pre-install wheels; either because the "
+            "increased cold boot time is is irrelevant to your use case or marginal compared to "
+            "other costs. Note that you may be able to use --max-install-jobs to decrease cold "
+            "boot times for some PEX deployment scenarios."
+        ),
+    )
+    group.add_argument(
+        "--max-install-jobs",
+        dest="max_install_jobs",
+        default=1,
+        type=int,
+        help=(
+            "The maximum number of parallel jobs to use when installing third party dependencies "
+            "contained in a PEX during its first boot. By default, this is set to 1 which "
+            "indicates dependencies should be installed in serial. A value of 2 or more indicates "
+            "dependencies should be installed in parallel using exactly this maximum number of "
+            "jobs. A value of 0 indicates the maximum number of parallel jobs should be "
+            "auto-selected taking the number of cores into account. Finally, a value of -1 "
+            "indicates the maximum number of parallel jobs should be auto-selected taking both the "
+            "characteristics of the third party dependencies contained in the PEX and the number "
+            "of cores into account. The third party dependency heuristics are intended to yield "
+            "good install performance, but are opaque and may change across PEX releases if better "
+            "heuristics are discovered. Any other value is illegal."
+        ),
+    )
     group.add_argument(
         "--check",
         dest="check",
@@ -824,6 +859,8 @@ def build_pex(
     pex_info.pex_root = options.runtime_pex_root
     pex_info.strip_pex_env = options.strip_pex_env
     pex_info.interpreter_constraints = interpreter_constraints
+    pex_info.deps_are_wheel_files = not options.pre_install_wheels
+    pex_info.max_install_jobs = options.max_install_jobs
 
     dependency_manager = DependencyManager()
     excluded = list(options.excluded)  # type: List[str]

pex/pex_info.py

@@ -105,6 +105,7 @@ def from_env(cls, env=ENV):
             "venv": Variables.PEX_VENV.strip_default(env),
             "inherit_path": inherit_path,
             "ignore_errors": Variables.PEX_IGNORE_ERRORS.strip_default(env),
+            "max_install_jobs": Variables.PEX_MAX_INSTALL_JOBS.strip_default(env),
         }
         # Filter out empty entries not explicitly set in the environment.
         return cls(info={k: v for k, v in pex_info.items() if v is not None})
@@ -503,6 +504,32 @@ def bootstrap_hash(self, value):
         # type: (str) -> None
         self._pex_info["bootstrap_hash"] = value
 
+    @property
+    def deps_are_wheel_files(self):
+        # type: () -> bool
+        return self._pex_info.get("deps_are_wheel_files", False)
+
+    @deps_are_wheel_files.setter
+    def deps_are_wheel_files(self, value):
+        # type: (bool) -> None
+        self._pex_info["deps_are_wheel_files"] = value
+
+    @property
+    def max_install_jobs(self):
+        # type: () -> int
+        return self._pex_info.get("max_install_jobs", 1)
+
+    @max_install_jobs.setter
+    def max_install_jobs(self, value):
+        # type: (int) -> None
+        if value < -1:
+            raise ValueError(
+                "The value for max_install_jobs must be -1 or greater; given: {jobs}".format(
+                    jobs=value
+                )
+            )
+        self._pex_info["max_install_jobs"] = value
+
     @property
     def bootstrap(self):
         # type: () -> str

pex/variables.py

@@ -603,7 +603,7 @@ def PEX_PYTHON_PATH(self):
         # type: () -> Optional[Tuple[str, ...]]
         """String.
 
-        A {pathsep!r} separated string containing paths of blessed Python interpreters for
+        A ':' or ';' separated string containing paths of blessed Python interpreters for
         overriding the Python interpreter used to invoke this PEX. Can be absolute paths to
         interpreters or standard $PATH style directory entries that are searched for child files
         that are python binaries.
@@ -617,7 +617,7 @@ def PEX_EXTRA_SYS_PATH(self):
         # type: () -> Tuple[str, ...]
         """String.
 
-        A {pathsep!r} separated string containing paths to add to the runtime sys.path.
+        A ':' or ';' separated string containing paths to add to the runtime sys.path.
 
         Should be used sparingly, e.g., if you know that code inside this PEX needs to
         interact with code outside it.
@@ -628,9 +628,7 @@ def PEX_EXTRA_SYS_PATH(self):
         existing sys.path (which you may not have control over) is scrubbed.
 
         See also PEX_PATH for how to merge packages from other pexes into the current environment.
-        """.format(
-            pathsep=os.pathsep
-        )
+        """
         return self._maybe_get_path_tuple("PEX_EXTRA_SYS_PATH") or ()
 
     @defaulted_property(default=os.path.join("~", ".pex"))
@@ -744,6 +742,37 @@ def PEX_TOOLS(self):
         """
         return self._get_bool("PEX_TOOLS")
 
+    @defaulted_property(default=1)
+    def PEX_MAX_INSTALL_JOBS(self):
+        # type: () -> int
+        """Integer.
+
+        The maximum number of parallel jobs to use when installing third party dependencies
+        contained in a PEX during its first boot. Values are interpreted as follows:
+
+        * ``>=2`` Dependencies should be installed in parallel using exactly this maximum number of
+          jobs.
+        * ``1`` Dependencies should be installed in serial.
+        * ``0`` The maximum number of parallel jobs should be auto-selected taking the number of
+          cores into account.
+        * ``-1`` The maximum number of parallel jobs should be auto-selected taking both the
+          characteristics of the third party dependencies contained in the PEX and the number of
+          cores into account. The third party dependency heuristics are intended to yield good
+          install performance, but are opaque and may change across PEX releases if better
+          heuristics are discovered.
+        * ``<=-2`` These are illegal values; an error is raised.
+
+        Default: 1
+        """
+        install_jobs = self._get_int("PEX_MAX_INSTALL_JOBS")
+        if install_jobs < -1:
+            raise ValueError(
+                "PEX_MAX_INSTALL_JOBS must be -1 or greater; given: {jobs}".format(
+                    jobs=install_jobs
+                )
+            )
+        return install_jobs
+
     def __repr__(self):
         return "{}({!r})".format(type(self).__name__, self._environ)