From 265a484efce787e7540de54be02b3cefc72a2f6d Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Sun, 27 Oct 2024 05:01:22 -0700 Subject: [PATCH] getdeps: add env subcommand Summary: X-link: https://github.com/facebookincubator/zstrong/pull/1025 Add getdeps.py `env` subcommand to get the environment from getdeps in a sourcable form so that its easier to run/test/debug the binaries There is an existing `debug` subcommand, but it starts an interactive shell inside getdeps, which isn't so useful for programatic use. To get the output clean enough to source I switched getdeps print() logging to go to stderr. example usage: ``` $ (source <(./build/fbcode_builder/getdeps.py --allow-system-packages env mononoke_integration); which mononoke) ``` Reviewed By: jdelliot Differential Revision: D64982397 fbshipit-source-id: 65212936d42185e4d395557b56d3dba499caa128 --- build/fbcode_builder/getdeps.py | 21 +++++++++++++++++ build/fbcode_builder/getdeps/builder.py | 30 ++++++++++++++++++++++-- build/fbcode_builder/getdeps/cargo.py | 23 ++++++++++++------ build/fbcode_builder/getdeps/manifest.py | 6 ++++- build/fbcode_builder/getdeps/runcmd.py | 8 ++----- 5 files changed, 72 insertions(+), 16 deletions(-) diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index ef81d3c3c88..01c6c1f68a8 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -923,6 +923,27 @@ def run_project_cmd(self, args, loader, manifest): self.create_builder(loader, manifest).debug(reconfigure=False) +@cmd( + "env", + "print the environment in a shell sourceable format", +) +class EnvCmd(ProjectCmdBase): + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--os-type", + help="Filter to just this OS type to run", + choices=["linux", "darwin", "windows"], + action="store", + dest="ostype", + default=None, + ) + + def run_project_cmd(self, args, loader, manifest): + if args.ostype: + loader.build_opts.host_type.ostype = args.ostype + self.create_builder(loader, manifest).printenv(reconfigure=False) + + @cmd("generate-github-actions", "generate a GitHub actions configuration") class GenerateGitHubActionsCmd(ProjectCmdBase): RUN_ON_ALL = """ [push, pull_request]""" diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 6ad9f2859a9..65bca2692a8 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -15,6 +15,7 @@ import subprocess import sys import typing +from shlex import quote as shellquote from typing import Optional from .dyndeps import create_dyn_dep_munger @@ -157,6 +158,29 @@ def debug(self, reconfigure: bool) -> None: shell = ["powershell.exe"] if sys.platform == "win32" else ["/bin/sh", "-i"] self._run_cmd(shell, cwd=self.build_dir, env=env) + def printenv(self, reconfigure: bool) -> None: + """print the environment in a shell sourcable format""" + reconfigure = self._reconfigure(reconfigure) + self._apply_patchfile() + self._prepare(reconfigure=reconfigure) + env = self._compute_env(env=Env(src={})) + prefix = "export " + sep = ":" + expand = "$" + expandpost = "" + if self.build_opts.is_windows(): + prefix = "SET " + sep = ";" + expand = "%" + expandpost = "%" + for k, v in sorted(env.items()): + existing = os.environ.get(k, None) + if k.endswith("PATH") and existing: + v = shellquote(v) + sep + f"{expand}{k}{expandpost}" + else: + v = shellquote(v) + print("%s%s=%s" % (prefix, k, v)) + def build(self, reconfigure: bool) -> None: print("Building %s..." % self.manifest.name) reconfigure = self._reconfigure(reconfigure) @@ -225,14 +249,16 @@ def _build(self, reconfigure) -> None: system needs to regenerate its rules.""" pass - def _compute_env(self): + def _compute_env(self, env=None) -> Env: + if env is None: + env = self.env # CMAKE_PREFIX_PATH is only respected when passed through the # environment, so we construct an appropriate path to pass down return self.build_opts.compute_env_for_install_dirs( self.loader, self.dep_manifests, self.ctx, - env=self.env, + env=env, manifest=self.manifest, ) diff --git a/build/fbcode_builder/getdeps/cargo.py b/build/fbcode_builder/getdeps/cargo.py index cae8bf54cac..0e0e0ddfe0b 100644 --- a/build/fbcode_builder/getdeps/cargo.py +++ b/build/fbcode_builder/getdeps/cargo.py @@ -9,6 +9,7 @@ import os import re import shutil +import sys import typing from .builder import BuilderBase @@ -97,7 +98,7 @@ def _create_cargo_config(self): if os.path.isfile(cargo_config_file): with open(cargo_config_file, "r") as f: - print(f"Reading {cargo_config_file}") + print(f"Reading {cargo_config_file}", file=sys.stderr) cargo_content = f.read() else: cargo_content = "" @@ -142,7 +143,8 @@ def _create_cargo_config(self): if new_content != cargo_content: with open(cargo_config_file, "w") as f: print( - f"Writing cargo config for {self.manifest.name} to {cargo_config_file}" + f"Writing cargo config for {self.manifest.name} to {cargo_config_file}", + file=sys.stderr, ) f.write(new_content) @@ -270,7 +272,10 @@ def _patchup_workspace(self, dep_to_git) -> None: new_content += "\n".join(config) if new_content != manifest_content: with open(patch_cargo, "w") as f: - print(f"writing patch to {patch_cargo}") + print( + f"writing patch to {patch_cargo}", + file=sys.stderr, + ) f.write(new_content) def _resolve_config(self, dep_to_git) -> typing.Dict[str, typing.Dict[str, str]]: @@ -296,7 +301,8 @@ def _resolve_config(self, dep_to_git) -> typing.Dict[str, typing.Dict[str, str]] if c in crate_source_map and c not in crates_to_patch_path: crates_to_patch_path[c] = crate_source_map[c] print( - f"{self.manifest.name}: Patching crate {c} via virtual manifest in {self.workspace_dir()}" + f"{self.manifest.name}: Patching crate {c} via virtual manifest in {self.workspace_dir()}", + file=sys.stderr, ) if crates_to_patch_path: git_url_to_crates_and_paths[git_url] = crates_to_patch_path @@ -352,7 +358,8 @@ def _resolve_dep_to_git(self): subpath = subpath.replace("/", "\\") crate_path = os.path.join(dep_source_dir, subpath) print( - f"{self.manifest.name}: Mapped crate {crate} to dep {dep} dir {crate_path}" + f"{self.manifest.name}: Mapped crate {crate} to dep {dep} dir {crate_path}", + file=sys.stderr, ) crate_source_map[crate] = crate_path elif dep_cargo_conf: @@ -367,7 +374,8 @@ def _resolve_dep_to_git(self): crate = match.group(1) if crate: print( - f"{self.manifest.name}: Discovered crate {crate} in dep {dep} dir {crate_root}" + f"{self.manifest.name}: Discovered crate {crate} in dep {dep} dir {crate_root}", + file=sys.stderr, ) crate_source_map[crate] = crate_root @@ -414,7 +422,8 @@ def _resolve_dep_to_crates(self, build_source_dir, dep_to_git): for c in crates: if c not in existing_crates: print( - f"Patch {self.manifest.name} uses {dep_name} crate {crates}" + f"Patch {self.manifest.name} uses {dep_name} crate {crates}", + file=sys.stderr, ) existing_crates.add(c) dep_to_crates.setdefault(name, set()).update(existing_crates) diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 6af5e3a74dc..d02cfe83f21 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -8,6 +8,7 @@ import configparser import io import os +import sys from typing import List from .builder import ( @@ -391,7 +392,10 @@ def _is_satisfied_by_preinstalled_environment(self, ctx): return False for key in envs: val = os.environ.get(key, None) - print(f"Testing ENV[{key}]: {repr(val)}") + print( + f"Testing ENV[{key}]: {repr(val)}", + file=sys.stderr, + ) if val is None: return False if len(val) == 0: diff --git a/build/fbcode_builder/getdeps/runcmd.py b/build/fbcode_builder/getdeps/runcmd.py index e0b9d2b22fd..c4a9326f9c2 100644 --- a/build/fbcode_builder/getdeps/runcmd.py +++ b/build/fbcode_builder/getdeps/runcmd.py @@ -10,16 +10,12 @@ import subprocess import sys +from shlex import quote as shellquote + from .envfuncs import Env from .platform import is_windows -try: - from shlex import quote as shellquote -except ImportError: - from pipes import quote as shellquote - - class RunCommandError(Exception): pass