From 4427a1fb0cf537da82e1ca4841e0cecda99e1107 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 6 Mar 2023 16:56:55 -0800 Subject: [PATCH] python: remove unused Java code With the deletion of the Java rule implementation, large amounts of code are now unused. Entire removal of the classes isn't yet possible, but most of their code can be removed. This also makes it easier to transition to the Starlark providers, since less Java code is referencing the to-be-deleted Java providers Work towards #15897 PiperOrigin-RevId: 514566571 Change-Id: I28ef124369fc678945873bc21919830cb73eafc8 --- .../bazel/rules/python/BazelPyBinaryRule.java | 12 - .../rules/python/BazelPyRuleClasses.java | 14 - .../bazel/rules/python/BazelPyTestRule.java | 22 - .../rules/python/BazelPythonSemantics.java | 461 +--------- .../build/lib/rules/python/PyCommon.java | 827 +----------------- 5 files changed, 13 insertions(+), 1323 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyBinaryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyBinaryRule.java index 43f72fc5966686..1327cdab47dbcb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyBinaryRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyBinaryRule.java @@ -14,13 +14,10 @@ package com.google.devtools.build.lib.bazel.rules.python; -import static com.google.devtools.build.lib.packages.Attribute.attr; -import static com.google.devtools.build.lib.packages.BuildType.LABEL; import com.google.devtools.build.lib.analysis.BaseRuleClasses; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; -import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory; import com.google.devtools.build.lib.bazel.rules.python.BazelPyRuleClasses.PyBinaryBaseRule; import com.google.devtools.build.lib.packages.RuleClass; import com.google.devtools.build.lib.rules.python.PyRuleClasses; @@ -41,15 +38,6 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) return builder .requiresConfigurationFragments(PythonConfiguration.class, BazelPythonConfiguration.class) .cfg(PyRuleClasses.VERSION_TRANSITION) - .add( - attr("$zipper", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .exec() - .value(env.getToolsLabel("//tools/zip:zipper"))) - .add( - attr("$launcher", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .value(env.getToolsLabel("//tools/launcher:launcher"))) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java index 3282c86bbe6f50..c238f4eb85389e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java @@ -17,7 +17,6 @@ import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.packages.BuildType.LABEL; import static com.google.devtools.build.lib.packages.BuildType.LABEL_LIST; -import static com.google.devtools.build.lib.packages.BuildType.NODEP_LABEL; import static com.google.devtools.build.lib.packages.BuildType.TRISTATE; import static com.google.devtools.build.lib.packages.Type.STRING; import static com.google.devtools.build.lib.packages.Type.STRING_LIST; @@ -36,7 +35,6 @@ import com.google.devtools.build.lib.rules.python.PyCommon; import com.google.devtools.build.lib.rules.python.PyInfo; import com.google.devtools.build.lib.rules.python.PyRuleClasses; -import com.google.devtools.build.lib.rules.python.PyRuntimeInfo; import com.google.devtools.build.lib.rules.python.PythonVersion; /** @@ -215,18 +213,6 @@ responsible for creating (possibly empty) __init__.py files and adding them to t

Stamped binaries are not rebuilt unless their dependencies change.

*/ .add(attr("stamp", TRISTATE).value(TriState.AUTO)) - // TODO(brandjon): Consider adding to py_interpreter a .mandatoryBuiltinProviders() of - // PyRuntimeInfoProvider. (Add a test case to PythonConfigurationTest for violations of - // this requirement.) Probably moot now that this is going to be replaced by toolchains. - .add(attr(":py_interpreter", LABEL).value(PY_INTERPRETER)) - .add( - attr("$py_toolchain_type", NODEP_LABEL) - .value(env.getToolsLabel("//tools/python:toolchain_type"))) - /* Only used when no py_runtime() is available. See #7901 - */ - .add( - attr("$default_bootstrap_template", LABEL) - .value(env.getToolsLabel(PyRuntimeInfo.DEFAULT_BOOTSTRAP_TEMPLATE))) .addToolchainTypes( ToolchainTypeRequirement.builder(env.getToolsLabel("//tools/python:toolchain_type")) .mandatory(true) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyTestRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyTestRule.java index 6075ece52fb33f..d2df05df645cda 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyTestRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyTestRule.java @@ -15,14 +15,12 @@ package com.google.devtools.build.lib.bazel.rules.python; import static com.google.devtools.build.lib.packages.Attribute.attr; -import static com.google.devtools.build.lib.packages.BuildType.LABEL; import static com.google.devtools.build.lib.packages.BuildType.TRISTATE; import static com.google.devtools.build.lib.packages.Type.BOOLEAN; import com.google.devtools.build.lib.analysis.BaseRuleClasses; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; -import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory; import com.google.devtools.build.lib.bazel.rules.python.BazelPyRuleClasses.PyBinaryBaseRule; import com.google.devtools.build.lib.packages.RuleClass; import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; @@ -39,11 +37,6 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) return builder .requiresConfigurationFragments(PythonConfiguration.class, BazelPythonConfiguration.class) .cfg(PyRuleClasses.VERSION_TRANSITION) - .add( - attr("$zipper", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .exec() - .value(env.getToolsLabel("//tools/zip:zipper"))) .override( attr("testonly", BOOLEAN) .value(true) @@ -53,21 +46,6 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) that the stamp argument is set to 0 by default for tests. */ .override(attr("stamp", TRISTATE).value(TriState.NO)) - .add( - attr("$launcher", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .value(env.getToolsLabel("//tools/launcher:launcher"))) - .add( - attr(":lcov_merger", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .value(BaseRuleClasses.getCoverageOutputGeneratorLabel())) - // Add the script as an attribute in order for py_test to output code coverage results for - // code covered by CC binaries invocations. - .add( - attr("$collect_cc_coverage", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .singleArtifact() - .value(env.getToolsLabel("//tools/test:collect_cc_coverage"))) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java index 56391e6cc4010b..f62111d45d82c1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java @@ -14,48 +14,22 @@ package com.google.devtools.build.lib.bazel.rules.python; -import static java.nio.charset.StandardCharsets.ISO_8859_1; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.CommandLineItem; -import com.google.devtools.build.lib.actions.ParamFileInfo; -import com.google.devtools.build.lib.actions.ParameterFile; -import com.google.devtools.build.lib.analysis.AnalysisUtils; -import com.google.devtools.build.lib.analysis.FilesToRunProvider; import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.Runfiles; import com.google.devtools.build.lib.analysis.RunfilesSupport; -import com.google.devtools.build.lib.analysis.ShToolchain; import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; -import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; -import com.google.devtools.build.lib.analysis.actions.LauncherFileWriteAction; -import com.google.devtools.build.lib.analysis.actions.LauncherFileWriteAction.LaunchInfo; -import com.google.devtools.build.lib.analysis.actions.SpawnAction; -import com.google.devtools.build.lib.analysis.actions.Substitution; -import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; -import com.google.devtools.build.lib.cmdline.LabelConstants; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.rules.cpp.CcInfo; -import com.google.devtools.build.lib.rules.python.PyCcLinkParamsProvider; import com.google.devtools.build.lib.rules.python.PyCommon; -import com.google.devtools.build.lib.rules.python.PyRuntimeInfo; -import com.google.devtools.build.lib.rules.python.PythonConfiguration; import com.google.devtools.build.lib.rules.python.PythonSemantics; import com.google.devtools.build.lib.rules.python.PythonUtils; -import com.google.devtools.build.lib.rules.python.PythonVersion; -import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.Serializable; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.function.Predicate; -import javax.annotation.Nullable; /** Functionality specific to the Python rules in Bazel. */ public class BazelPythonSemantics implements PythonSemantics { @@ -63,8 +37,6 @@ public class BazelPythonSemantics implements PythonSemantics { public static final Runfiles.EmptyFilesSupplier GET_INIT_PY_FILES = new PythonUtils.GetInitPyFiles((Predicate & Serializable) source -> false); - public static final PathFragment ZIP_RUNFILES_DIRECTORY_NAME = PathFragment.create("runfiles"); - @Override public Runfiles.EmptyFilesSupplier getEmptyRunfilesSupplier() { return GET_INIT_PY_FILES; @@ -72,245 +44,46 @@ public Runfiles.EmptyFilesSupplier getEmptyRunfilesSupplier() { @Override public String getSrcsVersionDocURL() { - // TODO(#8996): Update URL to point to rules_python's docs instead of the Bazel site. - return "https://bazel.build/reference/be/python#py_binary.srcs_version"; + throw new UnsupportedOperationException("Should not be called"); } @Override public void validate(RuleContext ruleContext, PyCommon common) { - PythonConfiguration config = ruleContext.getFragment(PythonConfiguration.class); - if (config.getDisablePy2()) { - var attrs = ruleContext.attributes(); - if (config.getDefaultPythonVersion().equals(PythonVersion.PY2) - || attrs.getOrDefault("python_version", Type.STRING, "UNSET").equals("PY2") - || attrs.getOrDefault("srcs_version", Type.STRING, "UNSET").equals("PY2") - || attrs.getOrDefault("srcs_version", Type.STRING, "UNSET").equals("PY2ONLY")) { - ruleContext.ruleError( - "Using Python 2 is not supported and disabled; see " - + "https://github.com/bazelbuild/bazel/issues/15684"); - return; - } - } + throw new UnsupportedOperationException("Should not be called"); } @Override public boolean prohibitHyphensInPackagePaths() { - return false; + throw new UnsupportedOperationException("Should not be called"); } @Override public void collectRunfilesForBinary( RuleContext ruleContext, Runfiles.Builder builder, PyCommon common, CcInfo ccInfo) { - addRuntime(ruleContext, common, builder); - // select() and build configuration should ideally remove coverage as - // as dependency, but guard against including it at runtime just in case. - if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { - addCoverageSupport(ruleContext, common, builder); - } + throw new UnsupportedOperationException("Should not be called"); } @Override public void collectDefaultRunfilesForBinary( RuleContext ruleContext, PyCommon common, Runfiles.Builder builder) { - addRuntime(ruleContext, common, builder); - if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { - addCoverageSupport(ruleContext, common, builder); - } + throw new UnsupportedOperationException("Should not be called"); } @Override public Collection precompiledPythonFiles( RuleContext ruleContext, Collection sources, PyCommon common) { - return ImmutableList.copyOf(sources); + throw new UnsupportedOperationException("Should not be called"); } @Override public List getImports(RuleContext ruleContext) { - List result = new ArrayList<>(); - PathFragment packageFragment = ruleContext.getLabel().getPackageIdentifier().getRunfilesPath(); - // Python scripts start with x.runfiles/ as the module space, so everything must be manually - // adjusted to be relative to the workspace name. - packageFragment = - PathFragment.create(ruleContext.getWorkspaceName()).getRelative(packageFragment); - for (String importsAttr : ruleContext.getExpander().list("imports")) { - if (importsAttr.startsWith("/")) { - ruleContext.attributeWarning( - "imports", "ignoring invalid absolute path '" + importsAttr + "'"); - continue; - } - PathFragment importsPath = packageFragment.getRelative(importsAttr); - if (importsPath.containsUplevelReferences()) { - ruleContext.attributeError( - "imports", "Path " + importsAttr + " references a path above the execution root"); - } - result.add(importsPath.getPathString()); - } - return result; - } - - private static String boolToLiteral(boolean value) { - return value ? "True" : "False"; - } - - private static String versionToLiteral(PythonVersion version) { - Preconditions.checkArgument(version.isTargetValue()); - return version == PythonVersion.PY3 ? "\"3\"" : "\"2\""; - } - - private static void createStubFile( - RuleContext ruleContext, Artifact stubOutput, PyCommon common, boolean isForZipFile) { - PythonConfiguration config = ruleContext.getFragment(PythonConfiguration.class); - BazelPythonConfiguration bazelConfig = ruleContext.getFragment(BazelPythonConfiguration.class); - - // The second-stage Python interpreter, which may be a system absolute path or a runfiles - // workspace-relative path. On Windows this is also passed to the launcher to use for the - // first-stage. - String pythonBinary = getPythonBinary(ruleContext, common, bazelConfig); - - // The python code coverage tool to use, if any. - String coverageTool = getCoverageTool(ruleContext, common); - - // Version information for exec config diagnostic warning. - PythonVersion attrVersion = PyCommon.readPythonVersionFromAttribute(ruleContext.attributes()); - boolean attrVersionSpecifiedExplicitly = attrVersion != null; - if (!attrVersionSpecifiedExplicitly) { - attrVersion = config.getDefaultPythonVersion(); - } - - Artifact bootstrapTemplate = getBootstrapTemplate(ruleContext, common); - - // Create the stub file. - ruleContext.registerAction( - new TemplateExpansionAction( - ruleContext.getActionOwner(), - bootstrapTemplate, - stubOutput, - ImmutableList.of( - Substitution.of("%shebang%", getStubShebang(ruleContext, common)), - Substitution.of("%main%", common.determineMainExecutableSource()), - Substitution.of("%python_binary%", pythonBinary), - Substitution.of("%coverage_tool%", coverageTool == null ? "" : coverageTool), - Substitution.of("%imports%", Joiner.on(":").join(common.getImports().toList())), - Substitution.of("%workspace_name%", ruleContext.getWorkspaceName()), - Substitution.of("%is_zipfile%", boolToLiteral(isForZipFile)), - Substitution.of( - "%import_all%", boolToLiteral(bazelConfig.getImportAllRepositories())), - Substitution.of("%target%", ruleContext.getRule().getLabel().getCanonicalForm()), - Substitution.of( - "%python_version_from_config%", versionToLiteral(common.getVersion())), - Substitution.of("%python_version_from_attr%", versionToLiteral(attrVersion)), - Substitution.of( - "%python_version_specified_explicitly%", - boolToLiteral(attrVersionSpecifiedExplicitly))), - true)); + throw new UnsupportedOperationException("Should not be called"); } @Override public void createExecutable( - RuleContext ruleContext, PyCommon common, CcInfo ccInfo, Runfiles.Builder runfilesBuilder) - throws InterruptedException { - PythonConfiguration config = ruleContext.getFragment(PythonConfiguration.class); - BazelPythonConfiguration bazelConfig = ruleContext.getFragment(BazelPythonConfiguration.class); - boolean buildPythonZip = config.buildPythonZip(); - - /* - * Python executable targets are launched in two stages. The first stage is the stub script that - * locates (and possibly extracts) the runfiles tree, sets up environment variables, and passes - * control to the second stage. The second stage is payload user code, i.e. the main Python - * file. - * - * When a zip file is built (--build_python_zip), the stub script becomes the __main__.py of the - * resulting zip, so that it runs when a Python interpreter executes the zip file. The stub - * logic will extract the zip's runfiles into a temporary directory. - * - * The stub script has a shebang pointing to a first-stage Python interpreter (as of this - * writing "#!/usr/bin/env python3"). When a zip file is built on unix, this shebang is also - * prepended to the final zip artifact. On Windows shebangs are ignored, and the launcher - * runs the first stage with an interpreter whose path is passed in as LaunchInfo. - */ - - // The initial entry point, which is the launcher on Windows, or the stub or zip file on Unix. - Artifact executable = common.getExecutable(); - - // The second-stage Python interpreter, which may be a system absolute path or a runfiles - // workspace-relative path. On Windows this is also passed to the launcher to use for the - // first-stage. - String pythonBinary = getPythonBinary(ruleContext, common, bazelConfig); - - // Create the stub file used for a non-zipfile executable. If --build_python_zip is true this is - // never used so we skip it. - if (!buildPythonZip) { - Artifact stubOutput = - OS.getCurrent() == OS.WINDOWS - ? common.getPythonStubArtifactForWindows(executable) - : executable; - createStubFile(ruleContext, stubOutput, common, /* isForZipFile= */ false); - } - - // Create the zip file if requested. On unix, copy it from the intermediate artifact to the - // final executable while prepending the shebang. - if (buildPythonZip) { - Artifact zipFile = common.getPythonZipArtifact(executable); - - // TODO(b/234923262): Take exec_group into consideration when selecting sh tools - if (OS.getCurrent() != OS.WINDOWS) { - PathFragment shExecutable = ShToolchain.getPathForHost(ruleContext.getConfiguration()); - String pythonExecutableName = "python3"; - // NOTE: keep the following line intact to support nix builds; nix patches - // this file to make it work for them; see https://github.com/bazelbuild/bazel/pull/11535 - String pythonShebang = "#!/usr/bin/env " + pythonExecutableName; - ruleContext.registerAction( - new SpawnAction.Builder() - .addInput(zipFile) - .addOutput(executable) - .setShellCommand( - shExecutable, - "echo '" - + pythonShebang - + "' | cat - " - + zipFile.getExecPathString() - + " > " - + executable.getExecPathString()) - .useDefaultShellEnvironment() - .setMnemonic("BuildBinary") - .build(ruleContext)); - } - } - - // On Windows, create the launcher. - if (OS.getCurrent() == OS.WINDOWS) { - createWindowsExeLauncher( - ruleContext, - // In the case where the second-stage interpreter is in runfiles, the launcher is passed - // a workspace-relative path that it combines with its own CWD to produce the full path to - // the real interpreter executable. (It can't use a path to the runfiles since they aren't - // yet extracted from the zip, assuming buildPythonZip is set.) - // - // TODO(#7947): Fix how this path is constructed for the case of a runfile interpreter in - // a remote repo -- probably need to pass an absolute path to the launcher instead of a - // workspace-relative one. Also ensure this is ok for remote execution, and if not, maybe - // change the launcher to use a separate system-installed first-stage interpreter like on - // unix. See also https://github.com/bazelbuild/bazel/issues/7947#issuecomment-491385802. - pythonBinary, - executable, - /* useZipFile= */ buildPythonZip); - } - } - - /** Registers an action to create a Windows Python launcher at {@code pythonLauncher}. */ - private static void createWindowsExeLauncher( - RuleContext ruleContext, String pythonBinary, Artifact pythonLauncher, boolean useZipFile) { - LaunchInfo launchInfo = - LaunchInfo.builder() - .addKeyValuePair("binary_type", "Python") - .addKeyValuePair("workspace_name", ruleContext.getWorkspaceName()) - .addKeyValuePair( - "symlink_runfiles_enabled", - ruleContext.getConfiguration().runfilesEnabled() ? "1" : "0") - .addKeyValuePair("python_bin_path", pythonBinary) - .addKeyValuePair("use_zip_file", useZipFile ? "1" : "0") - .build(); - LauncherFileWriteAction.createAndRegister(ruleContext, pythonLauncher, launchInfo); + RuleContext ruleContext, PyCommon common, CcInfo ccInfo, Runfiles.Builder runfilesBuilder) { + throw new UnsupportedOperationException("Should not be called"); } @Override @@ -319,224 +92,12 @@ public void postInitExecutable( RunfilesSupport runfilesSupport, PyCommon common, RuleConfiguredTargetBuilder builder) { - FilesToRunProvider zipper = ruleContext.getExecutablePrerequisite("$zipper"); - Artifact executable = common.getExecutable(); - Artifact zipFile = common.getPythonZipArtifact(executable); - - if (!ruleContext.hasErrors()) { - // Create the stub file that's needed by the python zip file. - Artifact stubFileForZipFile = common.getPythonIntermediateStubArtifact(executable); - createStubFile(ruleContext, stubFileForZipFile, common, /* isForZipFile= */ true); - - createPythonZipAction( - ruleContext, executable, zipFile, stubFileForZipFile, zipper, runfilesSupport); - } - builder.addOutputGroup("python_zip_file", zipFile); - } - - private static String getZipRunfilesPath( - PathFragment path, PathFragment workspaceName, boolean legacyExternalRunfiles) { - String zipRunfilesPath; - if (legacyExternalRunfiles && path.startsWith(LabelConstants.EXTERNAL_PATH_PREFIX)) { - // If the path starts with 'external' and --legacy_external_runfiles is set, this file is in - // an external repository. Convert it to the new runfiles path by removing the 'external' - // prefix. - zipRunfilesPath = path.relativeTo(LabelConstants.EXTERNAL_PATH_PREFIX).toString(); - } else { - // If not, it means the runfiles path is either under the workspace or an external file path - // in the new runfiles path format. In either case, simply appending it to the workspace name - // works just fine. - zipRunfilesPath = workspaceName.getRelative(path).toString(); - } - // We put the whole runfiles tree under the ZIP_RUNFILES_DIRECTORY_NAME directory, by doing this - // , we avoid the conflict between default workspace name "__main__" and __main__.py file. - // Note: This name has to be the same with the one in python_bootstrap_template.txt. - return ZIP_RUNFILES_DIRECTORY_NAME.getRelative(zipRunfilesPath).toString(); - } - - private static String getZipRunfilesPath( - String path, PathFragment workspaceName, boolean legacyExternalRunfiles) { - return getZipRunfilesPath(PathFragment.create(path), workspaceName, legacyExternalRunfiles); - } - - private static void createPythonZipAction( - RuleContext ruleContext, - Artifact executable, - Artifact zipFile, - Artifact stubFile, - FilesToRunProvider zipper, - RunfilesSupport runfilesSupport) { - - NestedSetBuilder inputsBuilder = NestedSetBuilder.stableOrder(); - PathFragment workspaceName = runfilesSupport.getWorkspaceName(); - CustomCommandLine.Builder argv = new CustomCommandLine.Builder(); - inputsBuilder.add(stubFile); - argv.addPrefixedExecPath("__main__.py=", stubFile); - boolean legacyExternalRunfiles = ruleContext.getConfiguration().legacyExternalRunfiles(); - - // Creating __init__.py files under each directory - argv.add("__init__.py="); - argv.addDynamicString( - getZipRunfilesPath("__init__.py", workspaceName, legacyExternalRunfiles) + "="); - for (String path : runfilesSupport.getRunfiles().getEmptyFilenames().toList()) { - argv.addDynamicString(getZipRunfilesPath(path, workspaceName, legacyExternalRunfiles) + "="); - } - - // Read each runfile from execute path, add them into zip file at the right runfiles path. - // Filter the executable file, cause we are building it. - argv.addAll( - CustomCommandLine.VectorArg.of(runfilesSupport.getRunfilesArtifacts()) - .mapped( - (CommandLineItem.CapturingMapFn) - (artifact, args) -> { - if (!artifact.equals(executable) && !artifact.equals(zipFile)) { - args.accept( - getZipRunfilesPath( - artifact.getRunfilesPath(), - workspaceName, - legacyExternalRunfiles) - + "=" - + artifact.getExecPathString()); - } - })); - - for (Artifact artifact : runfilesSupport.getRunfilesArtifacts().toList()) { - if (!artifact.equals(executable) && !artifact.equals(zipFile)) { - inputsBuilder.add(artifact); - } - } - - ruleContext.registerAction( - new SpawnAction.Builder() - .addTransitiveInputs(inputsBuilder.build()) - .addOutput(zipFile) - .setExecutable(zipper) - .useDefaultShellEnvironment() - .addCommandLine(CustomCommandLine.builder().add("cC").addExecPath(zipFile).build()) - // zipper can only consume file list options from param file not other options, - // so write file list in the param file. - .addCommandLine( - argv.build(), - ParamFileInfo.builder(ParameterFile.ParameterFileType.UNQUOTED) - .setCharset(ISO_8859_1) - .setUseAlways(true) - .build()) - .setMnemonic("PythonZipper") - .build(ruleContext)); - } - - /** - * Returns the Python runtime to use, either from the toolchain or the legacy flag-based - * mechanism. - * - *

Can only be called for an executable Python rule. - * - *

Returns {@code null} if there's a problem retrieving the runtime. - */ - @Nullable - private static PyRuntimeInfo getRuntime(RuleContext ruleContext, PyCommon common) { - return common.shouldGetRuntimeFromToolchain() - ? common.getRuntimeFromToolchain() - : ruleContext.getPrerequisite(":py_interpreter", PyRuntimeInfo.PROVIDER); - } - - private static Artifact getBootstrapTemplate(RuleContext ruleContext, PyCommon common) { - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null) { - Artifact bootstrapTemplate = provider.getBootstrapTemplate(); - if (bootstrapTemplate != null) { - return bootstrapTemplate; - } - } - return ruleContext.getPrerequisiteArtifact("$default_bootstrap_template"); - } - - private static void addRuntime( - RuleContext ruleContext, PyCommon common, Runfiles.Builder builder) { - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null && provider.isInBuild()) { - builder.addArtifact(provider.getInterpreter()); - // WARNING: we are adding the all Python runtime files here, - // and it would fail if the filenames of them contain spaces. - // Currently, we need to exclude them in py_runtime rules. - // Possible files in Python runtime which contain spaces in filenames: - // - https://github.com/pypa/setuptools/blob/master/setuptools/script%20(dev).tmpl - // - https://github.com/pypa/setuptools/blob/master/setuptools/command/launcher%20manifest.xml - builder.addTransitiveArtifacts(provider.getFiles()); - } - } - - private static String getPythonBinary( - RuleContext ruleContext, PyCommon common, BazelPythonConfiguration bazelConfig) { - String pythonBinary; - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null) { - // make use of py_runtime defined by --python_top - if (!provider.isInBuild()) { - // absolute Python path in py_runtime - pythonBinary = provider.getInterpreterPath().getPathString(); - } else { - // checked in Python interpreter in py_runtime - PathFragment workspaceName = - PathFragment.create(ruleContext.getRule().getPackage().getWorkspaceName()); - pythonBinary = - workspaceName.getRelative(provider.getInterpreter().getRunfilesPath()).getPathString(); - } - } else { - // make use of the Python interpreter in an absolute path - pythonBinary = bazelConfig.getPythonPath(); - } - - return pythonBinary; - } - - private static void addCoverageSupport( - RuleContext ruleContext, PyCommon common, Runfiles.Builder builder) { - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null && provider.getCoverageTool() != null) { - builder.addArtifact(provider.getCoverageTool()); - builder.addTransitiveArtifacts(provider.getCoverageToolFiles()); - } - } - - @Nullable - private static String getCoverageTool(RuleContext ruleContext, PyCommon common) { - if (!ruleContext.getConfiguration().isCodeCoverageEnabled()) { - return null; - } - String coverageTool = null; - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null && provider.getCoverageTool() != null) { - PathFragment workspaceName = - PathFragment.create(ruleContext.getRule().getPackage().getWorkspaceName()); - coverageTool = - workspaceName.getRelative(provider.getCoverageTool().getRunfilesPath()).getPathString(); - } - return coverageTool; - } - - private static String getStubShebang(RuleContext ruleContext, PyCommon common) { - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null) { - return provider.getStubShebang(); - } else { - return PyRuntimeInfo.DEFAULT_STUB_SHEBANG; - } + throw new UnsupportedOperationException("Should not be called"); } @Override public CcInfo buildCcInfoProvider( RuleContext ruleContext, Iterable deps) { - ImmutableList ccInfos = - ImmutableList.builder() - .addAll(AnalysisUtils.getProviders(deps, CcInfo.PROVIDER)) - .addAll( - AnalysisUtils.getProviders(deps, PyCcLinkParamsProvider.PROVIDER).stream() - .map(PyCcLinkParamsProvider::getCcInfo) - .collect(ImmutableList.toImmutableList())) - .build(); - - // TODO(plf): return empty CcInfo. - return CcInfo.merge(ccInfos); + throw new UnsupportedOperationException("Should not be called"); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java index 56c03f5764a5db..7d8663af09fa3a 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java @@ -13,65 +13,30 @@ // limitations under the License. package com.google.devtools.build.lib.rules.python; -import static net.starlark.java.eval.Starlark.NONE; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import com.google.devtools.build.lib.actions.Action; import com.google.devtools.build.lib.actions.ActionOwner; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.FailAction; import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; import com.google.devtools.build.lib.actions.extra.PythonInfo; -import com.google.devtools.build.lib.analysis.FileProvider; -import com.google.devtools.build.lib.analysis.OutputGroupInfo; import com.google.devtools.build.lib.analysis.PseudoAction; -import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.analysis.Runfiles; -import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; -import com.google.devtools.build.lib.analysis.Util; -import com.google.devtools.build.lib.analysis.platform.ToolchainInfo; -import com.google.devtools.build.lib.analysis.test.InstrumentedFilesCollector; -import com.google.devtools.build.lib.analysis.test.InstrumentedFilesCollector.InstrumentationSpec; -import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.packages.AttributeMap; -import com.google.devtools.build.lib.packages.BuildType; -import com.google.devtools.build.lib.packages.Rule; -import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.packages.Type; -import com.google.devtools.build.lib.rules.cpp.CcInfo; -import com.google.devtools.build.lib.rules.cpp.CppFileTypes; -import com.google.devtools.build.lib.server.FailureDetails.FailAction.Code; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; -import com.google.devtools.build.lib.util.FileType; -import com.google.devtools.build.lib.util.FileTypeSet; -import com.google.devtools.build.lib.util.OS; -import com.google.devtools.build.lib.vfs.PathFragment; import com.google.protobuf.GeneratedMessage.GeneratedExtension; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.UUID; import javax.annotation.Nullable; -import net.starlark.java.eval.EvalException; -import net.starlark.java.eval.Starlark; /** A helper class for analyzing a Python configured target. */ public final class PyCommon { - - private static final InstrumentationSpec INSTRUMENTATION_SPEC = - new InstrumentationSpec(FileTypeSet.of(PyRuleClasses.PYTHON_SOURCE)) - .withSourceAttributes("srcs") - .withDependencyAttributes("deps", "data"); - /** Name of the version attribute. */ public static final String PYTHON_VERSION_ATTRIBUTE = "python_version"; @@ -94,680 +59,7 @@ public static PythonVersion readPythonVersionFromAttribute(AttributeMap attrs) { return pythonVersionAttr != PythonVersion._INTERNAL_SENTINEL ? pythonVersionAttr : null; } - /** The context for the target this {@code PyCommon} is helping to analyze. */ - private final RuleContext ruleContext; - - /** The pluggable semantics object with hooks that customizes how analysis is done. */ - private final PythonSemantics semantics; - - /** - * The Python major version for which this target is being built, as per the {@code - * python_version} attribute or the configuration. - * - *

This is always either {@code PY2} or {@code PY3}. - */ - private final PythonVersion version; - - /** - * The level of compatibility with Python major versions, as per the {@code srcs_version} - * attribute. - */ - private final PythonVersion sourcesVersion; - - /** - * The Python sources belonging to this target's transitive {@code deps}, not including this - * target's own {@code srcs}. - */ - private final NestedSet dependencyTransitivePythonSources; - - /** - * The Python sources belonging to this target's transitive {@code deps}, including the Python - * sources in this target's {@code srcs}. - */ - private final NestedSet transitivePythonSources; - - /** - * The Python sources from this target's {@code srcs}. - * - *

This is computed slightly differently than the difference between {@link - * #dependencyTransitivePythonSources} and {@link transitivePythonSources}; it includes the - * filesToBuild rather than the prerequisite artifacts. - */ - // TODO(bazel-team): Can this be simplified to instead just be (transitivePythonSources - - // dependencyTransitivePythonSources)? - private final List directPythonSources; - - /** Whether this target or any of its {@code deps} or {@code data} deps has a shared library. */ - private final boolean usesSharedLibraries; - - /** Extra Python module import paths propagated or used by this target. */ - private final NestedSet imports; - - /** - * Whether any of this target's transitive {@code deps} have PY2-only source files, including this - * target itself. - */ - private final boolean hasPy2OnlySources; - - /** - * Whether any of this target's transitive {@code deps} have PY3-only source files, including this - * target itself. - */ - private final boolean hasPy3OnlySources; - - /** - * Information about the runtime, as obtained from the toolchain. - * - *

This is non-null only if - * - *

    - *
  1. the configuration says to pull the runtime from the toolchain (rather than from the - * legacy flags), - *
  2. the target defines the attribute "$py_toolchain_type" (in which case it MUST also declare - * that it requires the Python toolchain type), and - *
  3. we can successfully read the runtime info from the toolchain provider. - *
- */ - @Nullable private final PyRuntimeInfo runtimeFromToolchain; - - // Null if requiresMainFile is false, or the main artifact couldn't be determined. - @Nullable private Artifact mainArtifact = null; - - private Artifact executable = null; - - private NestedSet filesToBuild = null; - - private static String getOrderErrorMessage(String fieldName, Order expected, Order actual) { - return String.format( - "Incompatible order for %s: expected 'default' or '%s', got '%s'", - fieldName, expected.getStarlarkName(), actual.getStarlarkName()); - } - - // TODO(bazel-team): validateSources is the result of refactoring while preserving - // legacy behavior across some (but not all) Google-internal uses of PyCommon. Ideally all call - // sites should be updated to expect the same validation steps. - public PyCommon( - RuleContext ruleContext, - PythonSemantics semantics, - boolean validateSources, - boolean requiresMainFile) { - this.ruleContext = ruleContext; - this.semantics = semantics; - this.version = ruleContext.getFragment(PythonConfiguration.class).getPythonVersion(); - this.sourcesVersion = initSrcsVersionAttr(ruleContext); - this.dependencyTransitivePythonSources = initDependencyTransitivePythonSources(ruleContext); - this.transitivePythonSources = initTransitivePythonSources(ruleContext); - this.mainArtifact = requiresMainFile ? initMainArtifact(ruleContext) : null; - this.directPythonSources = - initAndMaybeValidateDirectPythonSources( - ruleContext, semantics, /* validate= */ validateSources, this.mainArtifact); - this.usesSharedLibraries = initUsesSharedLibraries(ruleContext); - this.imports = initImports(ruleContext, semantics); - this.hasPy2OnlySources = initHasPy2OnlySources(ruleContext, this.sourcesVersion); - this.hasPy3OnlySources = initHasPy3OnlySources(ruleContext, this.sourcesVersion); - this.runtimeFromToolchain = initRuntimeFromToolchain(ruleContext, this.version); - validatePythonVersionAttr(); - } - - /** Returns the parsed value of the "srcs_version" attribute. */ - private static PythonVersion initSrcsVersionAttr(RuleContext ruleContext) { - String attrValue = ruleContext.attributes().get("srcs_version", Type.STRING); - try { - return PythonVersion.parseSrcsValue(attrValue); - } catch (IllegalArgumentException ex) { - // Should already have been disallowed in the rule. - ruleContext.attributeError( - "srcs_version", - String.format( - "'%s' is not a valid value. Expected one of: %s", - attrValue, Joiner.on(", ").join(PythonVersion.SRCS_STRINGS))); - return PythonVersion.DEFAULT_SRCS_VALUE; - } - } - - private static NestedSet initDependencyTransitivePythonSources( - RuleContext ruleContext) { - NestedSetBuilder builder = NestedSetBuilder.compileOrder(); - collectTransitivePythonSourcesFromDeps(ruleContext, builder); - return builder.build(); - } - - private static NestedSet initTransitivePythonSources(RuleContext ruleContext) { - NestedSetBuilder builder = NestedSetBuilder.compileOrder(); - collectTransitivePythonSourcesFromDeps(ruleContext, builder); - builder.addAll( - ruleContext.getPrerequisiteArtifacts("srcs").filter(PyRuleClasses.PYTHON_SOURCE).list()); - return builder.build(); - } - - /** - * Gathers transitive .py files from {@code deps} (not including this target's {@code srcs} and - * adds them to {@code builder}. - * - *

If a target has the PyInfo provider, the value from that provider is used. Otherwise, we - * fall back on collecting .py source files from the target's filesToBuild. - */ - // TODO(bazel-team): Eliminate the fallback behavior by returning an appropriate py provider from - // the relevant rules. - private static void collectTransitivePythonSourcesFromDeps( - RuleContext ruleContext, NestedSetBuilder builder) { - for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps")) { - NestedSet sources; - if (dep.get(PyInfo.PROVIDER) != null) { - sources = dep.get(PyInfo.PROVIDER).getTransitiveSourcesSet(); - } else { - sources = - NestedSetBuilder.compileOrder() - .addAll( - FileType.filter( - dep.getProvider(FileProvider.class).getFilesToBuild().toList(), - PyRuleClasses.PYTHON_SOURCE)) - .build(); - } - builder.addTransitive(sources); - } - } - - private static List initAndMaybeValidateDirectPythonSources( - RuleContext ruleContext, PythonSemantics semantics, boolean validate, Artifact mainArtifact) { - List sourceFiles = new ArrayList<>(); - // TODO(bazel-team): Need to get the transitive deps closure, not just the sources of the rule. - for (TransitiveInfoCollection src : - ruleContext.getPrerequisitesIf("srcs", FileProvider.class)) { - // Make sure that none of the sources contain hyphens. - if (validate - && semantics.prohibitHyphensInPackagePaths() - && Util.containsHyphen(src.getLabel().getPackageFragment()) - // It's ok to have hyphens in main file - usually no one imports it. - && (mainArtifact == null || !src.getLabel().equals(mainArtifact.getOwnerLabel()))) { - ruleContext.attributeError( - "srcs", - src.getLabel() - + ": paths to Python sources (besides the main source) may not contain '-'"); - } - Iterable pySrcs = - FileType.filter( - src.getProvider(FileProvider.class).getFilesToBuild().toList(), - PyRuleClasses.PYTHON_SOURCE); - Iterables.addAll(sourceFiles, pySrcs); - if (validate && Iterables.isEmpty(pySrcs)) { - ruleContext.attributeWarning( - "srcs", "rule '" + src.getLabel() + "' does not produce any Python source files"); - } - } - return sourceFiles; - } - - /** - * Returns true if any of this target's {@code deps} or {@code data} deps has a shared library - * file (e.g. a {@code .so}) in its transitive dependency closure. - * - *

For targets with the py provider, we consult the {@code uses_shared_libraries} field. For - * targets without this provider, we look for {@link CppFileTypes#SHARED_LIBRARY}-type files in - * the filesToBuild. - */ - private static boolean initUsesSharedLibraries(RuleContext ruleContext) { - Iterable targets; - // The deps attribute must exist for all rule types that use PyCommon, but not necessarily the - // data attribute. - if (ruleContext.attributes().has("data")) { - targets = - Iterables.concat( - ruleContext.getPrerequisites("deps"), ruleContext.getPrerequisites("data")); - } else { - targets = ruleContext.getPrerequisites("deps"); - } - for (TransitiveInfoCollection target : targets) { - if (target.get(PyInfo.PROVIDER) != null) { - if (target.get(PyInfo.PROVIDER).getUsesSharedLibraries()) { - return true; - } - } else if (FileType.contains( - target.getProvider(FileProvider.class).getFilesToBuild().toList(), - CppFileTypes.SHARED_LIBRARY)) { - return true; - } - } - return false; - } - - /** - * Returns the transitive import paths of a target. - * - *

For targets with the PyInfo provider, the value from that provider is used. Otherwise, we - * default to an empty set. - */ - private static NestedSet initImports(RuleContext ruleContext, PythonSemantics semantics) { - NestedSetBuilder builder = NestedSetBuilder.compileOrder(); - builder.addAll(semantics.getImports(ruleContext)); - for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps")) { - NestedSet imports; - if (dep.get(PyInfo.PROVIDER) != null) { - imports = dep.get(PyInfo.PROVIDER).getImportsSet(); - } else { - imports = NestedSetBuilder.emptySet(Order.COMPILE_ORDER); - } - if (!builder.getOrder().isCompatible(imports.getOrder())) { - // TODO(brandjon): We should make order an invariant of the Python provider, and move this - // check into PyInfo. - ruleContext.ruleError( - getOrderErrorMessage("imports", builder.getOrder(), imports.getOrder())); - } else { - builder.addTransitive(imports); - } - } - return builder.build(); - } - - /** - * Returns true if any of {@code deps} has a py provider with {@code has_py2_only_sources} set, or - * this target has a {@code srcs_version} of {@code PY2} or {@code PY2ONLY}. - */ - private static boolean initHasPy2OnlySources( - RuleContext ruleContext, PythonVersion sourcesVersion) { - if (sourcesVersion == PythonVersion.PY2 || sourcesVersion == PythonVersion.PY2ONLY) { - return true; - } - for (PyInfo depInfo : ruleContext.getPrerequisites("deps", PyInfo.PROVIDER)) { - if (depInfo.getHasPy2OnlySources()) { - return true; - } - } - return false; - } - - /** - * Returns true if any of {@code deps} has a py provider with {@code has_py3_only_sources} set, or - * this target has {@code srcs_version} of {@code PY3} or {@code PY3ONLY}. - */ - private static boolean initHasPy3OnlySources( - RuleContext ruleContext, PythonVersion sourcesVersion) { - if (sourcesVersion == PythonVersion.PY3 || sourcesVersion == PythonVersion.PY3ONLY) { - return true; - } - for (PyInfo depInfo : ruleContext.getPrerequisites("deps", PyInfo.PROVIDER)) { - if (depInfo.getHasPy3OnlySources()) { - return true; - } - } - return false; - } - - /** - * Retrieves the {@link PyRuntimeInfo} object in the given field of the given {@link - * ToolchainInfo}. - * - *

If the field holds {@code None}, null is returned instead. - * - *

If the field does not exist on the given {@code ToolchainInfo}, or is not a {@code - * PyRuntimeInfo} and not {@code None}, an error is reported on the {@code ruleContext} and null - * is returned. - * - *

If the {@code PyRuntimeInfo} does not have {@code expectedVersion} as its Python version, an - * error is reported on the {@code ruleContext} (but the provider is still returned). - */ - @Nullable - private static PyRuntimeInfo parseRuntimeField( - RuleContext ruleContext, - PythonVersion expectedVersion, - ToolchainInfo toolchainInfo, - String field) { - Object fieldValue; - try { - fieldValue = toolchainInfo.getValue(field); - } catch (EvalException e) { - ruleContext.ruleError( - String.format( - "Error parsing the Python toolchain's ToolchainInfo: Could not retrieve field " - + "'%s': %s", - field, e.getMessage())); - return null; - } - if (fieldValue == null) { - ruleContext.ruleError( - String.format( - "Error parsing the Python toolchain's ToolchainInfo: field '%s' is missing", field)); - return null; - } - if (fieldValue == NONE) { - return null; - } - if (!(fieldValue instanceof PyRuntimeInfo)) { - ruleContext.ruleError( - String.format( - "Error parsing the Python toolchain's ToolchainInfo: Expected a PyRuntimeInfo in " - + "field '%s', but got '%s'", - field, Starlark.type(fieldValue))); - return null; - } - PyRuntimeInfo pyRuntimeInfo = (PyRuntimeInfo) fieldValue; - if (pyRuntimeInfo.getPythonVersion() != expectedVersion) { - ruleContext.ruleError( - String.format( - "Error retrieving the Python runtime from the toolchain: Expected field '%s' to have " - + "a runtime with python_version = '%s', but got python_version = '%s'", - field, expectedVersion.name(), pyRuntimeInfo.getPythonVersion().name())); - } - return pyRuntimeInfo; - } - - /** - * Returns a {@link PyRuntimeInfo} representing the runtime to use for this target, as retrieved - * from the resolved Python toolchain. - * - *

If the configuration says to use the legacy mechanism for obtaining the runtime rather than - * the toolchain mechanism, OR if this target's rule class does not define the - * "$py_toolchain_type" attribute, then null is returned. In this case no attempt is made to - * retrieve any toolchain information, and no errors are reported. - * - *

Otherwise, the toolchain provider structure is retrieved and validated, and any errors are - * reported on the rule context. If we're unable to determine the runtime due to an error, or if - * the toolchain does not specify a runtime for the version of Python we need, null is returned. - * - * @throws IllegalArgumentException if the rule class defines the "$py_toolchain_type" attribute - * but does not declare a requirement on the toolchain type - */ - @Nullable - private static PyRuntimeInfo initRuntimeFromToolchain( - RuleContext ruleContext, PythonVersion version) { - if (!shouldGetRuntimeFromToolchain(ruleContext) - || !ruleContext.attributes().has("$py_toolchain_type", BuildType.NODEP_LABEL)) { - return null; - } - Label toolchainType = ruleContext.attributes().get("$py_toolchain_type", BuildType.NODEP_LABEL); - ToolchainInfo toolchainInfo = ruleContext.getToolchainInfo(toolchainType); - Preconditions.checkArgument( - toolchainInfo != null, - "Could not retrieve a Python toolchain for '%s' rule", - ruleContext.getRule().getRuleClass()); - - PyRuntimeInfo py2RuntimeInfo = - parseRuntimeField(ruleContext, PythonVersion.PY2, toolchainInfo, "py2_runtime"); - PyRuntimeInfo py3RuntimeInfo = - parseRuntimeField(ruleContext, PythonVersion.PY3, toolchainInfo, "py3_runtime"); - Preconditions.checkState(version == PythonVersion.PY2 || version == PythonVersion.PY3); - PyRuntimeInfo result = version == PythonVersion.PY2 ? py2RuntimeInfo : py3RuntimeInfo; - if (result == null) { - ruleContext.ruleError( - String.format( - "The Python toolchain does not provide a runtime for Python version %s", - version.name())); - } - - // Hack around the fact that the autodetecting Python toolchain, which is automatically - // registered, does not yet support windows. In this case, we want to return null so that - // BazelPythonSemantics falls back on --python_path. See toolchain.bzl. - // TODO(#7844): Remove this hack when the autodetecting toolchain has a windows implementation. - if (py2RuntimeInfo != null - && py2RuntimeInfo.getInterpreterPathString() != null - && py2RuntimeInfo - .getInterpreterPathString() - .equals("/_magic_pyruntime_sentinel_do_not_use")) { - return null; - } - - return result; - } - - /** - * Reports an attribute error if {@code python_version} cannot be parsed as {@code PY2}, {@code - * PY3}, or the sentinel value. - * - *

This *should* be enforced by rule attribute validation ({@link - * Attribute.Builder.allowedValues}), but this check is here to fail-fast just in case. - */ - private void validatePythonVersionAttr() { - AttributeMap attrs = ruleContext.attributes(); - if (!attrs.has(PYTHON_VERSION_ATTRIBUTE, Type.STRING)) { - return; - } - String attrValue = attrs.get(PYTHON_VERSION_ATTRIBUTE, Type.STRING); - try { - PythonVersion.parseTargetOrSentinelValue(attrValue); - } catch (IllegalArgumentException ex) { - ruleContext.attributeError( - PYTHON_VERSION_ATTRIBUTE, - String.format("'%s' is not a valid value. Expected either 'PY2' or 'PY3'", attrValue)); - } - } - - /** - * If the Python version (as determined by the configuration) is inconsistent with {@link - * #hasPy2OnlySources} or {@link #hasPy3OnlySources}, emits a {@link FailAction} that "generates" - * the executable. - * - *

We use a {@code FailAction} rather than a rule error because we want to defer the error - * until the execution phase. This way, we still get a configured target that the user can query - * over with an aspect to find the exact transitive dependency that introduced the offending - * version constraint. (See {@code //tools/python/srcs_version.bzl%find_requirements}) - * - * @return true if a {@link FailAction} was created - */ - private boolean maybeCreateFailActionDueToTransitiveSourcesVersion() { - String errorTemplate = - ruleContext.getLabel() - + ": " - + "This target is being built for Python %s but (transitively) includes Python %s-only " - + "sources. You can get diagnostic information about which dependencies introduce this " - + "version requirement by running the `find_requirements` aspect. For more info see " - + "the documentation for the `srcs_version` attribute: " - + semantics.getSrcsVersionDocURL(); - - String error = null; - if (version == PythonVersion.PY2 && hasPy3OnlySources) { - error = String.format(errorTemplate, "2", "3"); - } else if (version == PythonVersion.PY3 && hasPy2OnlySources) { - error = String.format(errorTemplate, "3", "2"); - } - if (error == null) { - return false; - } else { - ruleContext.registerAction( - new FailAction( - ruleContext.getActionOwner(), - ImmutableList.of(executable), - error, - Code.INCORRECT_PYTHON_VERSION)); - return true; - } - } - - public PythonVersion getVersion() { - return version; - } - - public PythonVersion getSourcesVersion() { - return sourcesVersion; - } - - /** - * Returns the transitive Python sources collected from the deps attribute, not including sources - * from the srcs attribute (unless they were separately reached via deps). - */ - public NestedSet getDependencyTransitivePythonSources() { - return dependencyTransitivePythonSources; - } - - /** Returns the transitive Python sources collected from the deps and srcs attributes. */ - public NestedSet getTransitivePythonSources() { - return transitivePythonSources; - } - - public boolean usesSharedLibraries() { - return usesSharedLibraries; - } - - public NestedSet getImports() { - return imports; - } - - public boolean hasPy2OnlySources() { - return hasPy2OnlySources; - } - - public boolean hasPy3OnlySources() { - return hasPy3OnlySources; - } - - /** - * Returns {@code true} if the Python runtime should be obtained from the Python toolchain (as per - * {@code --incompatible_use_python_toolchains}), as opposed to through the legacy mechanism - * specified in the {@link PythonSemantics} (e.g., {@code --python_top}). - */ - public boolean shouldGetRuntimeFromToolchain() { - return shouldGetRuntimeFromToolchain(ruleContext); - } - - private static boolean shouldGetRuntimeFromToolchain(RuleContext ruleContext) { - return ruleContext.getFragment(PythonConfiguration.class).useToolchains(); - } - - /** - * Returns a {@link PyRuntimeInfo} representing the runtime to use for this target, as retrieved - * from the resolved toolchain. - * - *

This may only be called for executable Python rules (rules defining the attribute - * "$py_toolchain_type", i.e. {@code py_binary} and {@code py_test}). In addition, it may not be - * called if {@link #shouldGetRuntimeFromToolchain()} returns false. - * - *

If there was a problem retrieving the runtime information from the toolchain, null is - * returned. An error would have already been reported on the rule context at {@code PyCommon} - * initialization time. - */ - @Nullable - public PyRuntimeInfo getRuntimeFromToolchain() { - Preconditions.checkArgument( - ruleContext.attributes().has("$py_toolchain_type", BuildType.NODEP_LABEL), - "Cannot retrieve Python toolchain information for '%s' rule", - ruleContext.getRule().getRuleClass()); - Preconditions.checkArgument( - shouldGetRuntimeFromToolchain(), - "Access to the Python toolchain is disabled by --incompatible_use_python_toolchains=false"); - - return runtimeFromToolchain; - } - - public void initBinary(List srcs) { - Preconditions.checkNotNull(version); - - if (OS.getCurrent() == OS.WINDOWS) { - executable = - ruleContext.getImplicitOutputArtifact(ruleContext.getTarget().getName() + ".exe"); - } else { - executable = ruleContext.createOutputArtifact(); - } - - NestedSetBuilder filesToBuildBuilder = - NestedSetBuilder.stableOrder().addAll(srcs).add(executable); - - if (ruleContext.getFragment(PythonConfiguration.class).buildPythonZip()) { - filesToBuildBuilder.add(getPythonZipArtifact(executable)); - } else if (OS.getCurrent() == OS.WINDOWS) { - // TODO(bazel-team): Here we should check target platform instead of using OS.getCurrent(). - // On Windows, add the python stub launcher in the set of files to build. - filesToBuildBuilder.add(getPythonStubArtifactForWindows(executable)); - } - - filesToBuild = filesToBuildBuilder.build(); - - if (ruleContext.hasErrors()) { - return; - } - - addPyExtraActionPseudoAction(); - } - - /** - * @return an artifact next to the executable file with a given suffix. - */ - private Artifact getArtifactWithExtension(Artifact executable, String extension) { - // On Windows, the Python executable has .exe extension on Windows, - // On Linux, the Python executable has no extension. - // We can't use ruleContext#getRelatedArtifact because it would mangle files with dots in the - // name on non-Windows platforms. - PathFragment pathFragment = - executable.getOutputDirRelativePath( - ruleContext.getConfiguration().isSiblingRepositoryLayout()); - String fileName = executable.getFilename(); - if (OS.getCurrent() == OS.WINDOWS) { - Preconditions.checkArgument(fileName.endsWith(".exe")); - fileName = fileName.substring(0, fileName.length() - 4) + extension; - } else { - fileName = fileName + extension; - } - return ruleContext.getDerivedArtifact(pathFragment.replaceName(fileName), executable.getRoot()); - } - - /** Returns an artifact next to the executable file with ".zip" suffix. */ - public Artifact getPythonZipArtifact(Artifact executable) { - return getArtifactWithExtension(executable, ".zip"); - } - - /** - * Returns an artifact next to the executable file with ".temp" suffix. Used only if we're - * building a zip. - */ - public Artifact getPythonIntermediateStubArtifact(Artifact executable) { - return getArtifactWithExtension(executable, ".temp"); - } - - /** Returns an artifact next to the executable file with no suffix. Only called for Windows. */ - public Artifact getPythonStubArtifactForWindows(Artifact executable) { - return ruleContext.getRelatedArtifact( - executable.getOutputDirRelativePath( - ruleContext.getConfiguration().isSiblingRepositoryLayout()), - ""); - } - - /** - * Adds a PyInfo provider. - * - *

This is a public method because some rules just want a PyInfo provider without the other - * things py_library needs. - */ - public void addPyInfoProvider(RuleConfiguredTargetBuilder builder) { - builder.addNativeDeclaredProvider( - PyInfo.builder() - .setTransitiveSources(transitivePythonSources) - .setUsesSharedLibraries(usesSharedLibraries) - .setImports(imports) - .setHasPy2OnlySources(hasPy2OnlySources) - .setHasPy3OnlySources(hasPy3OnlySources) - .build()); - } - - public void addCommonTransitiveInfoProviders(RuleConfiguredTargetBuilder builder) { - addPyInfoProvider(builder); - - // Add PyRuntimeInfo if this is an executable rule. - if (runtimeFromToolchain != null) { - builder.addNativeDeclaredProvider(runtimeFromToolchain); - } - - builder - .addNativeDeclaredProvider( - InstrumentedFilesCollector.collect(ruleContext, INSTRUMENTATION_SPEC)) - // Python targets are not really compilable. The best we can do is make sure that all - // generated source files are ready. - .addOutputGroup(OutputGroupInfo.FILES_TO_COMPILE, transitivePythonSources) - .addOutputGroup(OutputGroupInfo.COMPILATION_PREREQUISITES, transitivePythonSources); - } - - /** Returns a list of the source artifacts */ - public List getPythonSources() { - return directPythonSources; - } - - /** - * Adds a {@link PseudoAction} to the build graph that is only used for providing information to - * the blaze extra_action feature. - */ - void addPyExtraActionPseudoAction() { - if (ruleContext.getConfiguration().getActionListeners().isEmpty()) { - return; - } - registerPyExtraActionPseudoAction(ruleContext, dependencyTransitivePythonSources); - } + private PyCommon() {} // Public so that Starlark bindings can access it. Should only be called by PyStarlarkBuiltins. // TODO(b/253059598): Remove support for this; https://github.com/bazelbuild/bazel/issues/16455 @@ -788,7 +80,7 @@ public static void registerPyExtraActionPseudoAction( * Creates a {@link PseudoAction} that is only used for providing information to the blaze * extra_action feature. */ - public static Action makePyExtraActionPseudoAction( + private static Action makePyExtraActionPseudoAction( ActionOwner owner, List sources, NestedSet dependencies, @@ -815,121 +107,6 @@ public static Action makePyExtraActionPseudoAction( @SerializationConstant @AutoCodec.VisibleForSerialization static final GeneratedExtension PYTHON_INFO = PythonInfo.pythonInfo; - /** Returns the artifact for the main Python source file. */ - private static Artifact initMainArtifact(RuleContext ruleContext) { - String mainSourceName; - Rule target = ruleContext.getRule(); - boolean explicitMain = target.isAttributeValueExplicitlySpecified("main"); - if (explicitMain) { - mainSourceName = ruleContext.attributes().get("main", BuildType.LABEL).getName(); - if (!mainSourceName.endsWith(".py")) { - ruleContext.attributeError("main", "main must end in '.py'"); - } - } else { - String ruleName = target.getName(); - // TODO(blaze-team) Shouldn't the same check apply for all Python targets? - if (ruleName.endsWith(".py")) { - ruleContext.attributeError("name", "name must not end in '.py'"); - } - mainSourceName = ruleName + ".py"; - } - PathFragment mainSourcePath = PathFragment.create(mainSourceName); - - Artifact mainArtifact = null; - for (Artifact outItem : ruleContext.getPrerequisiteArtifacts("srcs").list()) { - if (outItem.getRootRelativePath().endsWith(mainSourcePath)) { - if (mainArtifact == null) { - mainArtifact = outItem; - } else { - ruleContext.attributeError( - "srcs", - buildMultipleMainMatchesErrorText( - explicitMain, - mainSourceName, - mainArtifact.getRunfilesPath().toString(), - outItem.getRunfilesPath().toString())); - } - } - } - - if (mainArtifact == null) { - ruleContext.attributeError("srcs", buildNoMainMatchesErrorText(explicitMain, mainSourceName)); - } - - return mainArtifact; - } - - /** - * Returns the path to the main python entry point, or null if this PyCommon was initialized with - * {@code requiresMainFile} set to false (or there was an error in determining the main artifact). - */ - @Nullable - public String determineMainExecutableSource() { - if (mainArtifact == null) { - return null; - } - PathFragment workspaceName = - PathFragment.create(ruleContext.getRule().getPackage().getWorkspaceName()); - return workspaceName.getRelative(mainArtifact.getRunfilesPath()).getPathString(); - } - - public Artifact getExecutable() { - return executable; - } - - public NestedSet getFilesToBuild() { - return filesToBuild; - } - - /** - * Creates the actual executable artifact, i.e., emits a generating action for {@link - * #getExecutable()}. - * - *

If there is a transitive sources version conflict, may produce a {@link FailAction} to - * trigger an execution-time failure. See {@link - * #maybeCreateFailActionDueToTransitiveSourcesVersion}. - */ - public void createExecutable(CcInfo ccInfo, Runfiles.Builder defaultRunfilesBuilder) - throws InterruptedException, RuleErrorException { - boolean failed = maybeCreateFailActionDueToTransitiveSourcesVersion(); - if (!failed) { - semantics.createExecutable(ruleContext, this, ccInfo, defaultRunfilesBuilder); - } - } - - private static String buildMultipleMainMatchesErrorText( - boolean explicit, String proposedMainName, String match1, String match2) { - String errorText; - if (explicit) { - errorText = - String.format( - "file name '%s' specified by 'main' attribute matches multiple files: e.g., '%s' and" - + " '%s'", - proposedMainName, match1, match2); - } else { - errorText = - String.format( - "default main file name '%s' matches multiple files. Perhaps specify an explicit" - + " file with 'main' attribute? Matches were: '%s' and '%s'", - proposedMainName, match1, match2); - } - return errorText; - } - - private static String buildNoMainMatchesErrorText(boolean explicit, String proposedMainName) { - String errorText; - if (explicit) { - errorText = "could not find '" + proposedMainName + "' as specified by 'main' attribute"; - } else { - errorText = - String.format( - "corresponding default '%s' does not appear in srcs. Add it or override default file" - + " name with a 'main' attribute", - proposedMainName); - } - return errorText; - } - // Used purely to set the legacy ActionType of the ExtraActionInfo. @Immutable private static final class PyPseudoAction extends PseudoAction {