diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyBinaryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyBinaryRule.java index 43f72fc5966686..1327cdab47dbcb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyBinaryRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyBinaryRule.java @@ -14,13 +14,10 @@ package com.google.devtools.build.lib.bazel.rules.python; -import static com.google.devtools.build.lib.packages.Attribute.attr; -import static com.google.devtools.build.lib.packages.BuildType.LABEL; import com.google.devtools.build.lib.analysis.BaseRuleClasses; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; -import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory; import com.google.devtools.build.lib.bazel.rules.python.BazelPyRuleClasses.PyBinaryBaseRule; import com.google.devtools.build.lib.packages.RuleClass; import com.google.devtools.build.lib.rules.python.PyRuleClasses; @@ -41,15 +38,6 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) return builder .requiresConfigurationFragments(PythonConfiguration.class, BazelPythonConfiguration.class) .cfg(PyRuleClasses.VERSION_TRANSITION) - .add( - attr("$zipper", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .exec() - .value(env.getToolsLabel("//tools/zip:zipper"))) - .add( - attr("$launcher", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .value(env.getToolsLabel("//tools/launcher:launcher"))) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java index 3282c86bbe6f50..c238f4eb85389e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java @@ -17,7 +17,6 @@ import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.packages.BuildType.LABEL; import static com.google.devtools.build.lib.packages.BuildType.LABEL_LIST; -import static com.google.devtools.build.lib.packages.BuildType.NODEP_LABEL; import static com.google.devtools.build.lib.packages.BuildType.TRISTATE; import static com.google.devtools.build.lib.packages.Type.STRING; import static com.google.devtools.build.lib.packages.Type.STRING_LIST; @@ -36,7 +35,6 @@ import com.google.devtools.build.lib.rules.python.PyCommon; import com.google.devtools.build.lib.rules.python.PyInfo; import com.google.devtools.build.lib.rules.python.PyRuleClasses; -import com.google.devtools.build.lib.rules.python.PyRuntimeInfo; import com.google.devtools.build.lib.rules.python.PythonVersion; /** @@ -215,18 +213,6 @@ responsible for creating (possibly empty) __init__.py files and adding them to t

Stamped binaries are not rebuilt unless their dependencies change.

*/ .add(attr("stamp", TRISTATE).value(TriState.AUTO)) - // TODO(brandjon): Consider adding to py_interpreter a .mandatoryBuiltinProviders() of - // PyRuntimeInfoProvider. (Add a test case to PythonConfigurationTest for violations of - // this requirement.) Probably moot now that this is going to be replaced by toolchains. - .add(attr(":py_interpreter", LABEL).value(PY_INTERPRETER)) - .add( - attr("$py_toolchain_type", NODEP_LABEL) - .value(env.getToolsLabel("//tools/python:toolchain_type"))) - /* Only used when no py_runtime() is available. See #7901 - */ - .add( - attr("$default_bootstrap_template", LABEL) - .value(env.getToolsLabel(PyRuntimeInfo.DEFAULT_BOOTSTRAP_TEMPLATE))) .addToolchainTypes( ToolchainTypeRequirement.builder(env.getToolsLabel("//tools/python:toolchain_type")) .mandatory(true) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyTestRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyTestRule.java index 6075ece52fb33f..d2df05df645cda 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyTestRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyTestRule.java @@ -15,14 +15,12 @@ package com.google.devtools.build.lib.bazel.rules.python; import static com.google.devtools.build.lib.packages.Attribute.attr; -import static com.google.devtools.build.lib.packages.BuildType.LABEL; import static com.google.devtools.build.lib.packages.BuildType.TRISTATE; import static com.google.devtools.build.lib.packages.Type.BOOLEAN; import com.google.devtools.build.lib.analysis.BaseRuleClasses; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; -import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory; import com.google.devtools.build.lib.bazel.rules.python.BazelPyRuleClasses.PyBinaryBaseRule; import com.google.devtools.build.lib.packages.RuleClass; import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; @@ -39,11 +37,6 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) return builder .requiresConfigurationFragments(PythonConfiguration.class, BazelPythonConfiguration.class) .cfg(PyRuleClasses.VERSION_TRANSITION) - .add( - attr("$zipper", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .exec() - .value(env.getToolsLabel("//tools/zip:zipper"))) .override( attr("testonly", BOOLEAN) .value(true) @@ -53,21 +46,6 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) that the stamp argument is set to 0 by default for tests. */ .override(attr("stamp", TRISTATE).value(TriState.NO)) - .add( - attr("$launcher", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .value(env.getToolsLabel("//tools/launcher:launcher"))) - .add( - attr(":lcov_merger", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .value(BaseRuleClasses.getCoverageOutputGeneratorLabel())) - // Add the script as an attribute in order for py_test to output code coverage results for - // code covered by CC binaries invocations. - .add( - attr("$collect_cc_coverage", LABEL) - .cfg(ExecutionTransitionFactory.create()) - .singleArtifact() - .value(env.getToolsLabel("//tools/test:collect_cc_coverage"))) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java index 56391e6cc4010b..f62111d45d82c1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java @@ -14,48 +14,22 @@ package com.google.devtools.build.lib.bazel.rules.python; -import static java.nio.charset.StandardCharsets.ISO_8859_1; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.CommandLineItem; -import com.google.devtools.build.lib.actions.ParamFileInfo; -import com.google.devtools.build.lib.actions.ParameterFile; -import com.google.devtools.build.lib.analysis.AnalysisUtils; -import com.google.devtools.build.lib.analysis.FilesToRunProvider; import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.Runfiles; import com.google.devtools.build.lib.analysis.RunfilesSupport; -import com.google.devtools.build.lib.analysis.ShToolchain; import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; -import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; -import com.google.devtools.build.lib.analysis.actions.LauncherFileWriteAction; -import com.google.devtools.build.lib.analysis.actions.LauncherFileWriteAction.LaunchInfo; -import com.google.devtools.build.lib.analysis.actions.SpawnAction; -import com.google.devtools.build.lib.analysis.actions.Substitution; -import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; -import com.google.devtools.build.lib.cmdline.LabelConstants; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.rules.cpp.CcInfo; -import com.google.devtools.build.lib.rules.python.PyCcLinkParamsProvider; import com.google.devtools.build.lib.rules.python.PyCommon; -import com.google.devtools.build.lib.rules.python.PyRuntimeInfo; -import com.google.devtools.build.lib.rules.python.PythonConfiguration; import com.google.devtools.build.lib.rules.python.PythonSemantics; import com.google.devtools.build.lib.rules.python.PythonUtils; -import com.google.devtools.build.lib.rules.python.PythonVersion; -import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.Serializable; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.function.Predicate; -import javax.annotation.Nullable; /** Functionality specific to the Python rules in Bazel. */ public class BazelPythonSemantics implements PythonSemantics { @@ -63,8 +37,6 @@ public class BazelPythonSemantics implements PythonSemantics { public static final Runfiles.EmptyFilesSupplier GET_INIT_PY_FILES = new PythonUtils.GetInitPyFiles((Predicate & Serializable) source -> false); - public static final PathFragment ZIP_RUNFILES_DIRECTORY_NAME = PathFragment.create("runfiles"); - @Override public Runfiles.EmptyFilesSupplier getEmptyRunfilesSupplier() { return GET_INIT_PY_FILES; @@ -72,245 +44,46 @@ public Runfiles.EmptyFilesSupplier getEmptyRunfilesSupplier() { @Override public String getSrcsVersionDocURL() { - // TODO(#8996): Update URL to point to rules_python's docs instead of the Bazel site. - return "https://bazel.build/reference/be/python#py_binary.srcs_version"; + throw new UnsupportedOperationException("Should not be called"); } @Override public void validate(RuleContext ruleContext, PyCommon common) { - PythonConfiguration config = ruleContext.getFragment(PythonConfiguration.class); - if (config.getDisablePy2()) { - var attrs = ruleContext.attributes(); - if (config.getDefaultPythonVersion().equals(PythonVersion.PY2) - || attrs.getOrDefault("python_version", Type.STRING, "UNSET").equals("PY2") - || attrs.getOrDefault("srcs_version", Type.STRING, "UNSET").equals("PY2") - || attrs.getOrDefault("srcs_version", Type.STRING, "UNSET").equals("PY2ONLY")) { - ruleContext.ruleError( - "Using Python 2 is not supported and disabled; see " - + "https://github.com/bazelbuild/bazel/issues/15684"); - return; - } - } + throw new UnsupportedOperationException("Should not be called"); } @Override public boolean prohibitHyphensInPackagePaths() { - return false; + throw new UnsupportedOperationException("Should not be called"); } @Override public void collectRunfilesForBinary( RuleContext ruleContext, Runfiles.Builder builder, PyCommon common, CcInfo ccInfo) { - addRuntime(ruleContext, common, builder); - // select() and build configuration should ideally remove coverage as - // as dependency, but guard against including it at runtime just in case. - if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { - addCoverageSupport(ruleContext, common, builder); - } + throw new UnsupportedOperationException("Should not be called"); } @Override public void collectDefaultRunfilesForBinary( RuleContext ruleContext, PyCommon common, Runfiles.Builder builder) { - addRuntime(ruleContext, common, builder); - if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { - addCoverageSupport(ruleContext, common, builder); - } + throw new UnsupportedOperationException("Should not be called"); } @Override public Collection precompiledPythonFiles( RuleContext ruleContext, Collection sources, PyCommon common) { - return ImmutableList.copyOf(sources); + throw new UnsupportedOperationException("Should not be called"); } @Override public List getImports(RuleContext ruleContext) { - List result = new ArrayList<>(); - PathFragment packageFragment = ruleContext.getLabel().getPackageIdentifier().getRunfilesPath(); - // Python scripts start with x.runfiles/ as the module space, so everything must be manually - // adjusted to be relative to the workspace name. - packageFragment = - PathFragment.create(ruleContext.getWorkspaceName()).getRelative(packageFragment); - for (String importsAttr : ruleContext.getExpander().list("imports")) { - if (importsAttr.startsWith("/")) { - ruleContext.attributeWarning( - "imports", "ignoring invalid absolute path '" + importsAttr + "'"); - continue; - } - PathFragment importsPath = packageFragment.getRelative(importsAttr); - if (importsPath.containsUplevelReferences()) { - ruleContext.attributeError( - "imports", "Path " + importsAttr + " references a path above the execution root"); - } - result.add(importsPath.getPathString()); - } - return result; - } - - private static String boolToLiteral(boolean value) { - return value ? "True" : "False"; - } - - private static String versionToLiteral(PythonVersion version) { - Preconditions.checkArgument(version.isTargetValue()); - return version == PythonVersion.PY3 ? "\"3\"" : "\"2\""; - } - - private static void createStubFile( - RuleContext ruleContext, Artifact stubOutput, PyCommon common, boolean isForZipFile) { - PythonConfiguration config = ruleContext.getFragment(PythonConfiguration.class); - BazelPythonConfiguration bazelConfig = ruleContext.getFragment(BazelPythonConfiguration.class); - - // The second-stage Python interpreter, which may be a system absolute path or a runfiles - // workspace-relative path. On Windows this is also passed to the launcher to use for the - // first-stage. - String pythonBinary = getPythonBinary(ruleContext, common, bazelConfig); - - // The python code coverage tool to use, if any. - String coverageTool = getCoverageTool(ruleContext, common); - - // Version information for exec config diagnostic warning. - PythonVersion attrVersion = PyCommon.readPythonVersionFromAttribute(ruleContext.attributes()); - boolean attrVersionSpecifiedExplicitly = attrVersion != null; - if (!attrVersionSpecifiedExplicitly) { - attrVersion = config.getDefaultPythonVersion(); - } - - Artifact bootstrapTemplate = getBootstrapTemplate(ruleContext, common); - - // Create the stub file. - ruleContext.registerAction( - new TemplateExpansionAction( - ruleContext.getActionOwner(), - bootstrapTemplate, - stubOutput, - ImmutableList.of( - Substitution.of("%shebang%", getStubShebang(ruleContext, common)), - Substitution.of("%main%", common.determineMainExecutableSource()), - Substitution.of("%python_binary%", pythonBinary), - Substitution.of("%coverage_tool%", coverageTool == null ? "" : coverageTool), - Substitution.of("%imports%", Joiner.on(":").join(common.getImports().toList())), - Substitution.of("%workspace_name%", ruleContext.getWorkspaceName()), - Substitution.of("%is_zipfile%", boolToLiteral(isForZipFile)), - Substitution.of( - "%import_all%", boolToLiteral(bazelConfig.getImportAllRepositories())), - Substitution.of("%target%", ruleContext.getRule().getLabel().getCanonicalForm()), - Substitution.of( - "%python_version_from_config%", versionToLiteral(common.getVersion())), - Substitution.of("%python_version_from_attr%", versionToLiteral(attrVersion)), - Substitution.of( - "%python_version_specified_explicitly%", - boolToLiteral(attrVersionSpecifiedExplicitly))), - true)); + throw new UnsupportedOperationException("Should not be called"); } @Override public void createExecutable( - RuleContext ruleContext, PyCommon common, CcInfo ccInfo, Runfiles.Builder runfilesBuilder) - throws InterruptedException { - PythonConfiguration config = ruleContext.getFragment(PythonConfiguration.class); - BazelPythonConfiguration bazelConfig = ruleContext.getFragment(BazelPythonConfiguration.class); - boolean buildPythonZip = config.buildPythonZip(); - - /* - * Python executable targets are launched in two stages. The first stage is the stub script that - * locates (and possibly extracts) the runfiles tree, sets up environment variables, and passes - * control to the second stage. The second stage is payload user code, i.e. the main Python - * file. - * - * When a zip file is built (--build_python_zip), the stub script becomes the __main__.py of the - * resulting zip, so that it runs when a Python interpreter executes the zip file. The stub - * logic will extract the zip's runfiles into a temporary directory. - * - * The stub script has a shebang pointing to a first-stage Python interpreter (as of this - * writing "#!/usr/bin/env python3"). When a zip file is built on unix, this shebang is also - * prepended to the final zip artifact. On Windows shebangs are ignored, and the launcher - * runs the first stage with an interpreter whose path is passed in as LaunchInfo. - */ - - // The initial entry point, which is the launcher on Windows, or the stub or zip file on Unix. - Artifact executable = common.getExecutable(); - - // The second-stage Python interpreter, which may be a system absolute path or a runfiles - // workspace-relative path. On Windows this is also passed to the launcher to use for the - // first-stage. - String pythonBinary = getPythonBinary(ruleContext, common, bazelConfig); - - // Create the stub file used for a non-zipfile executable. If --build_python_zip is true this is - // never used so we skip it. - if (!buildPythonZip) { - Artifact stubOutput = - OS.getCurrent() == OS.WINDOWS - ? common.getPythonStubArtifactForWindows(executable) - : executable; - createStubFile(ruleContext, stubOutput, common, /* isForZipFile= */ false); - } - - // Create the zip file if requested. On unix, copy it from the intermediate artifact to the - // final executable while prepending the shebang. - if (buildPythonZip) { - Artifact zipFile = common.getPythonZipArtifact(executable); - - // TODO(b/234923262): Take exec_group into consideration when selecting sh tools - if (OS.getCurrent() != OS.WINDOWS) { - PathFragment shExecutable = ShToolchain.getPathForHost(ruleContext.getConfiguration()); - String pythonExecutableName = "python3"; - // NOTE: keep the following line intact to support nix builds; nix patches - // this file to make it work for them; see https://github.com/bazelbuild/bazel/pull/11535 - String pythonShebang = "#!/usr/bin/env " + pythonExecutableName; - ruleContext.registerAction( - new SpawnAction.Builder() - .addInput(zipFile) - .addOutput(executable) - .setShellCommand( - shExecutable, - "echo '" - + pythonShebang - + "' | cat - " - + zipFile.getExecPathString() - + " > " - + executable.getExecPathString()) - .useDefaultShellEnvironment() - .setMnemonic("BuildBinary") - .build(ruleContext)); - } - } - - // On Windows, create the launcher. - if (OS.getCurrent() == OS.WINDOWS) { - createWindowsExeLauncher( - ruleContext, - // In the case where the second-stage interpreter is in runfiles, the launcher is passed - // a workspace-relative path that it combines with its own CWD to produce the full path to - // the real interpreter executable. (It can't use a path to the runfiles since they aren't - // yet extracted from the zip, assuming buildPythonZip is set.) - // - // TODO(#7947): Fix how this path is constructed for the case of a runfile interpreter in - // a remote repo -- probably need to pass an absolute path to the launcher instead of a - // workspace-relative one. Also ensure this is ok for remote execution, and if not, maybe - // change the launcher to use a separate system-installed first-stage interpreter like on - // unix. See also https://github.com/bazelbuild/bazel/issues/7947#issuecomment-491385802. - pythonBinary, - executable, - /* useZipFile= */ buildPythonZip); - } - } - - /** Registers an action to create a Windows Python launcher at {@code pythonLauncher}. */ - private static void createWindowsExeLauncher( - RuleContext ruleContext, String pythonBinary, Artifact pythonLauncher, boolean useZipFile) { - LaunchInfo launchInfo = - LaunchInfo.builder() - .addKeyValuePair("binary_type", "Python") - .addKeyValuePair("workspace_name", ruleContext.getWorkspaceName()) - .addKeyValuePair( - "symlink_runfiles_enabled", - ruleContext.getConfiguration().runfilesEnabled() ? "1" : "0") - .addKeyValuePair("python_bin_path", pythonBinary) - .addKeyValuePair("use_zip_file", useZipFile ? "1" : "0") - .build(); - LauncherFileWriteAction.createAndRegister(ruleContext, pythonLauncher, launchInfo); + RuleContext ruleContext, PyCommon common, CcInfo ccInfo, Runfiles.Builder runfilesBuilder) { + throw new UnsupportedOperationException("Should not be called"); } @Override @@ -319,224 +92,12 @@ public void postInitExecutable( RunfilesSupport runfilesSupport, PyCommon common, RuleConfiguredTargetBuilder builder) { - FilesToRunProvider zipper = ruleContext.getExecutablePrerequisite("$zipper"); - Artifact executable = common.getExecutable(); - Artifact zipFile = common.getPythonZipArtifact(executable); - - if (!ruleContext.hasErrors()) { - // Create the stub file that's needed by the python zip file. - Artifact stubFileForZipFile = common.getPythonIntermediateStubArtifact(executable); - createStubFile(ruleContext, stubFileForZipFile, common, /* isForZipFile= */ true); - - createPythonZipAction( - ruleContext, executable, zipFile, stubFileForZipFile, zipper, runfilesSupport); - } - builder.addOutputGroup("python_zip_file", zipFile); - } - - private static String getZipRunfilesPath( - PathFragment path, PathFragment workspaceName, boolean legacyExternalRunfiles) { - String zipRunfilesPath; - if (legacyExternalRunfiles && path.startsWith(LabelConstants.EXTERNAL_PATH_PREFIX)) { - // If the path starts with 'external' and --legacy_external_runfiles is set, this file is in - // an external repository. Convert it to the new runfiles path by removing the 'external' - // prefix. - zipRunfilesPath = path.relativeTo(LabelConstants.EXTERNAL_PATH_PREFIX).toString(); - } else { - // If not, it means the runfiles path is either under the workspace or an external file path - // in the new runfiles path format. In either case, simply appending it to the workspace name - // works just fine. - zipRunfilesPath = workspaceName.getRelative(path).toString(); - } - // We put the whole runfiles tree under the ZIP_RUNFILES_DIRECTORY_NAME directory, by doing this - // , we avoid the conflict between default workspace name "__main__" and __main__.py file. - // Note: This name has to be the same with the one in python_bootstrap_template.txt. - return ZIP_RUNFILES_DIRECTORY_NAME.getRelative(zipRunfilesPath).toString(); - } - - private static String getZipRunfilesPath( - String path, PathFragment workspaceName, boolean legacyExternalRunfiles) { - return getZipRunfilesPath(PathFragment.create(path), workspaceName, legacyExternalRunfiles); - } - - private static void createPythonZipAction( - RuleContext ruleContext, - Artifact executable, - Artifact zipFile, - Artifact stubFile, - FilesToRunProvider zipper, - RunfilesSupport runfilesSupport) { - - NestedSetBuilder inputsBuilder = NestedSetBuilder.stableOrder(); - PathFragment workspaceName = runfilesSupport.getWorkspaceName(); - CustomCommandLine.Builder argv = new CustomCommandLine.Builder(); - inputsBuilder.add(stubFile); - argv.addPrefixedExecPath("__main__.py=", stubFile); - boolean legacyExternalRunfiles = ruleContext.getConfiguration().legacyExternalRunfiles(); - - // Creating __init__.py files under each directory - argv.add("__init__.py="); - argv.addDynamicString( - getZipRunfilesPath("__init__.py", workspaceName, legacyExternalRunfiles) + "="); - for (String path : runfilesSupport.getRunfiles().getEmptyFilenames().toList()) { - argv.addDynamicString(getZipRunfilesPath(path, workspaceName, legacyExternalRunfiles) + "="); - } - - // Read each runfile from execute path, add them into zip file at the right runfiles path. - // Filter the executable file, cause we are building it. - argv.addAll( - CustomCommandLine.VectorArg.of(runfilesSupport.getRunfilesArtifacts()) - .mapped( - (CommandLineItem.CapturingMapFn) - (artifact, args) -> { - if (!artifact.equals(executable) && !artifact.equals(zipFile)) { - args.accept( - getZipRunfilesPath( - artifact.getRunfilesPath(), - workspaceName, - legacyExternalRunfiles) - + "=" - + artifact.getExecPathString()); - } - })); - - for (Artifact artifact : runfilesSupport.getRunfilesArtifacts().toList()) { - if (!artifact.equals(executable) && !artifact.equals(zipFile)) { - inputsBuilder.add(artifact); - } - } - - ruleContext.registerAction( - new SpawnAction.Builder() - .addTransitiveInputs(inputsBuilder.build()) - .addOutput(zipFile) - .setExecutable(zipper) - .useDefaultShellEnvironment() - .addCommandLine(CustomCommandLine.builder().add("cC").addExecPath(zipFile).build()) - // zipper can only consume file list options from param file not other options, - // so write file list in the param file. - .addCommandLine( - argv.build(), - ParamFileInfo.builder(ParameterFile.ParameterFileType.UNQUOTED) - .setCharset(ISO_8859_1) - .setUseAlways(true) - .build()) - .setMnemonic("PythonZipper") - .build(ruleContext)); - } - - /** - * Returns the Python runtime to use, either from the toolchain or the legacy flag-based - * mechanism. - * - *

Can only be called for an executable Python rule. - * - *

Returns {@code null} if there's a problem retrieving the runtime. - */ - @Nullable - private static PyRuntimeInfo getRuntime(RuleContext ruleContext, PyCommon common) { - return common.shouldGetRuntimeFromToolchain() - ? common.getRuntimeFromToolchain() - : ruleContext.getPrerequisite(":py_interpreter", PyRuntimeInfo.PROVIDER); - } - - private static Artifact getBootstrapTemplate(RuleContext ruleContext, PyCommon common) { - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null) { - Artifact bootstrapTemplate = provider.getBootstrapTemplate(); - if (bootstrapTemplate != null) { - return bootstrapTemplate; - } - } - return ruleContext.getPrerequisiteArtifact("$default_bootstrap_template"); - } - - private static void addRuntime( - RuleContext ruleContext, PyCommon common, Runfiles.Builder builder) { - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null && provider.isInBuild()) { - builder.addArtifact(provider.getInterpreter()); - // WARNING: we are adding the all Python runtime files here, - // and it would fail if the filenames of them contain spaces. - // Currently, we need to exclude them in py_runtime rules. - // Possible files in Python runtime which contain spaces in filenames: - // - https://github.com/pypa/setuptools/blob/master/setuptools/script%20(dev).tmpl - // - https://github.com/pypa/setuptools/blob/master/setuptools/command/launcher%20manifest.xml - builder.addTransitiveArtifacts(provider.getFiles()); - } - } - - private static String getPythonBinary( - RuleContext ruleContext, PyCommon common, BazelPythonConfiguration bazelConfig) { - String pythonBinary; - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null) { - // make use of py_runtime defined by --python_top - if (!provider.isInBuild()) { - // absolute Python path in py_runtime - pythonBinary = provider.getInterpreterPath().getPathString(); - } else { - // checked in Python interpreter in py_runtime - PathFragment workspaceName = - PathFragment.create(ruleContext.getRule().getPackage().getWorkspaceName()); - pythonBinary = - workspaceName.getRelative(provider.getInterpreter().getRunfilesPath()).getPathString(); - } - } else { - // make use of the Python interpreter in an absolute path - pythonBinary = bazelConfig.getPythonPath(); - } - - return pythonBinary; - } - - private static void addCoverageSupport( - RuleContext ruleContext, PyCommon common, Runfiles.Builder builder) { - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null && provider.getCoverageTool() != null) { - builder.addArtifact(provider.getCoverageTool()); - builder.addTransitiveArtifacts(provider.getCoverageToolFiles()); - } - } - - @Nullable - private static String getCoverageTool(RuleContext ruleContext, PyCommon common) { - if (!ruleContext.getConfiguration().isCodeCoverageEnabled()) { - return null; - } - String coverageTool = null; - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null && provider.getCoverageTool() != null) { - PathFragment workspaceName = - PathFragment.create(ruleContext.getRule().getPackage().getWorkspaceName()); - coverageTool = - workspaceName.getRelative(provider.getCoverageTool().getRunfilesPath()).getPathString(); - } - return coverageTool; - } - - private static String getStubShebang(RuleContext ruleContext, PyCommon common) { - PyRuntimeInfo provider = getRuntime(ruleContext, common); - if (provider != null) { - return provider.getStubShebang(); - } else { - return PyRuntimeInfo.DEFAULT_STUB_SHEBANG; - } + throw new UnsupportedOperationException("Should not be called"); } @Override public CcInfo buildCcInfoProvider( RuleContext ruleContext, Iterable deps) { - ImmutableList ccInfos = - ImmutableList.builder() - .addAll(AnalysisUtils.getProviders(deps, CcInfo.PROVIDER)) - .addAll( - AnalysisUtils.getProviders(deps, PyCcLinkParamsProvider.PROVIDER).stream() - .map(PyCcLinkParamsProvider::getCcInfo) - .collect(ImmutableList.toImmutableList())) - .build(); - - // TODO(plf): return empty CcInfo. - return CcInfo.merge(ccInfos); + throw new UnsupportedOperationException("Should not be called"); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java index 56c03f5764a5db..7d8663af09fa3a 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java @@ -13,65 +13,30 @@ // limitations under the License. package com.google.devtools.build.lib.rules.python; -import static net.starlark.java.eval.Starlark.NONE; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import com.google.devtools.build.lib.actions.Action; import com.google.devtools.build.lib.actions.ActionOwner; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.FailAction; import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; import com.google.devtools.build.lib.actions.extra.PythonInfo; -import com.google.devtools.build.lib.analysis.FileProvider; -import com.google.devtools.build.lib.analysis.OutputGroupInfo; import com.google.devtools.build.lib.analysis.PseudoAction; -import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.analysis.Runfiles; -import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; -import com.google.devtools.build.lib.analysis.Util; -import com.google.devtools.build.lib.analysis.platform.ToolchainInfo; -import com.google.devtools.build.lib.analysis.test.InstrumentedFilesCollector; -import com.google.devtools.build.lib.analysis.test.InstrumentedFilesCollector.InstrumentationSpec; -import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.packages.AttributeMap; -import com.google.devtools.build.lib.packages.BuildType; -import com.google.devtools.build.lib.packages.Rule; -import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.packages.Type; -import com.google.devtools.build.lib.rules.cpp.CcInfo; -import com.google.devtools.build.lib.rules.cpp.CppFileTypes; -import com.google.devtools.build.lib.server.FailureDetails.FailAction.Code; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; -import com.google.devtools.build.lib.util.FileType; -import com.google.devtools.build.lib.util.FileTypeSet; -import com.google.devtools.build.lib.util.OS; -import com.google.devtools.build.lib.vfs.PathFragment; import com.google.protobuf.GeneratedMessage.GeneratedExtension; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.UUID; import javax.annotation.Nullable; -import net.starlark.java.eval.EvalException; -import net.starlark.java.eval.Starlark; /** A helper class for analyzing a Python configured target. */ public final class PyCommon { - - private static final InstrumentationSpec INSTRUMENTATION_SPEC = - new InstrumentationSpec(FileTypeSet.of(PyRuleClasses.PYTHON_SOURCE)) - .withSourceAttributes("srcs") - .withDependencyAttributes("deps", "data"); - /** Name of the version attribute. */ public static final String PYTHON_VERSION_ATTRIBUTE = "python_version"; @@ -94,680 +59,7 @@ public static PythonVersion readPythonVersionFromAttribute(AttributeMap attrs) { return pythonVersionAttr != PythonVersion._INTERNAL_SENTINEL ? pythonVersionAttr : null; } - /** The context for the target this {@code PyCommon} is helping to analyze. */ - private final RuleContext ruleContext; - - /** The pluggable semantics object with hooks that customizes how analysis is done. */ - private final PythonSemantics semantics; - - /** - * The Python major version for which this target is being built, as per the {@code - * python_version} attribute or the configuration. - * - *

This is always either {@code PY2} or {@code PY3}. - */ - private final PythonVersion version; - - /** - * The level of compatibility with Python major versions, as per the {@code srcs_version} - * attribute. - */ - private final PythonVersion sourcesVersion; - - /** - * The Python sources belonging to this target's transitive {@code deps}, not including this - * target's own {@code srcs}. - */ - private final NestedSet dependencyTransitivePythonSources; - - /** - * The Python sources belonging to this target's transitive {@code deps}, including the Python - * sources in this target's {@code srcs}. - */ - private final NestedSet transitivePythonSources; - - /** - * The Python sources from this target's {@code srcs}. - * - *

This is computed slightly differently than the difference between {@link - * #dependencyTransitivePythonSources} and {@link transitivePythonSources}; it includes the - * filesToBuild rather than the prerequisite artifacts. - */ - // TODO(bazel-team): Can this be simplified to instead just be (transitivePythonSources - - // dependencyTransitivePythonSources)? - private final List directPythonSources; - - /** Whether this target or any of its {@code deps} or {@code data} deps has a shared library. */ - private final boolean usesSharedLibraries; - - /** Extra Python module import paths propagated or used by this target. */ - private final NestedSet imports; - - /** - * Whether any of this target's transitive {@code deps} have PY2-only source files, including this - * target itself. - */ - private final boolean hasPy2OnlySources; - - /** - * Whether any of this target's transitive {@code deps} have PY3-only source files, including this - * target itself. - */ - private final boolean hasPy3OnlySources; - - /** - * Information about the runtime, as obtained from the toolchain. - * - *

This is non-null only if - * - *

    - *
  1. the configuration says to pull the runtime from the toolchain (rather than from the - * legacy flags), - *
  2. the target defines the attribute "$py_toolchain_type" (in which case it MUST also declare - * that it requires the Python toolchain type), and - *
  3. we can successfully read the runtime info from the toolchain provider. - *
- */ - @Nullable private final PyRuntimeInfo runtimeFromToolchain; - - // Null if requiresMainFile is false, or the main artifact couldn't be determined. - @Nullable private Artifact mainArtifact = null; - - private Artifact executable = null; - - private NestedSet filesToBuild = null; - - private static String getOrderErrorMessage(String fieldName, Order expected, Order actual) { - return String.format( - "Incompatible order for %s: expected 'default' or '%s', got '%s'", - fieldName, expected.getStarlarkName(), actual.getStarlarkName()); - } - - // TODO(bazel-team): validateSources is the result of refactoring while preserving - // legacy behavior across some (but not all) Google-internal uses of PyCommon. Ideally all call - // sites should be updated to expect the same validation steps. - public PyCommon( - RuleContext ruleContext, - PythonSemantics semantics, - boolean validateSources, - boolean requiresMainFile) { - this.ruleContext = ruleContext; - this.semantics = semantics; - this.version = ruleContext.getFragment(PythonConfiguration.class).getPythonVersion(); - this.sourcesVersion = initSrcsVersionAttr(ruleContext); - this.dependencyTransitivePythonSources = initDependencyTransitivePythonSources(ruleContext); - this.transitivePythonSources = initTransitivePythonSources(ruleContext); - this.mainArtifact = requiresMainFile ? initMainArtifact(ruleContext) : null; - this.directPythonSources = - initAndMaybeValidateDirectPythonSources( - ruleContext, semantics, /* validate= */ validateSources, this.mainArtifact); - this.usesSharedLibraries = initUsesSharedLibraries(ruleContext); - this.imports = initImports(ruleContext, semantics); - this.hasPy2OnlySources = initHasPy2OnlySources(ruleContext, this.sourcesVersion); - this.hasPy3OnlySources = initHasPy3OnlySources(ruleContext, this.sourcesVersion); - this.runtimeFromToolchain = initRuntimeFromToolchain(ruleContext, this.version); - validatePythonVersionAttr(); - } - - /** Returns the parsed value of the "srcs_version" attribute. */ - private static PythonVersion initSrcsVersionAttr(RuleContext ruleContext) { - String attrValue = ruleContext.attributes().get("srcs_version", Type.STRING); - try { - return PythonVersion.parseSrcsValue(attrValue); - } catch (IllegalArgumentException ex) { - // Should already have been disallowed in the rule. - ruleContext.attributeError( - "srcs_version", - String.format( - "'%s' is not a valid value. Expected one of: %s", - attrValue, Joiner.on(", ").join(PythonVersion.SRCS_STRINGS))); - return PythonVersion.DEFAULT_SRCS_VALUE; - } - } - - private static NestedSet initDependencyTransitivePythonSources( - RuleContext ruleContext) { - NestedSetBuilder builder = NestedSetBuilder.compileOrder(); - collectTransitivePythonSourcesFromDeps(ruleContext, builder); - return builder.build(); - } - - private static NestedSet initTransitivePythonSources(RuleContext ruleContext) { - NestedSetBuilder builder = NestedSetBuilder.compileOrder(); - collectTransitivePythonSourcesFromDeps(ruleContext, builder); - builder.addAll( - ruleContext.getPrerequisiteArtifacts("srcs").filter(PyRuleClasses.PYTHON_SOURCE).list()); - return builder.build(); - } - - /** - * Gathers transitive .py files from {@code deps} (not including this target's {@code srcs} and - * adds them to {@code builder}. - * - *

If a target has the PyInfo provider, the value from that provider is used. Otherwise, we - * fall back on collecting .py source files from the target's filesToBuild. - */ - // TODO(bazel-team): Eliminate the fallback behavior by returning an appropriate py provider from - // the relevant rules. - private static void collectTransitivePythonSourcesFromDeps( - RuleContext ruleContext, NestedSetBuilder builder) { - for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps")) { - NestedSet sources; - if (dep.get(PyInfo.PROVIDER) != null) { - sources = dep.get(PyInfo.PROVIDER).getTransitiveSourcesSet(); - } else { - sources = - NestedSetBuilder.compileOrder() - .addAll( - FileType.filter( - dep.getProvider(FileProvider.class).getFilesToBuild().toList(), - PyRuleClasses.PYTHON_SOURCE)) - .build(); - } - builder.addTransitive(sources); - } - } - - private static List initAndMaybeValidateDirectPythonSources( - RuleContext ruleContext, PythonSemantics semantics, boolean validate, Artifact mainArtifact) { - List sourceFiles = new ArrayList<>(); - // TODO(bazel-team): Need to get the transitive deps closure, not just the sources of the rule. - for (TransitiveInfoCollection src : - ruleContext.getPrerequisitesIf("srcs", FileProvider.class)) { - // Make sure that none of the sources contain hyphens. - if (validate - && semantics.prohibitHyphensInPackagePaths() - && Util.containsHyphen(src.getLabel().getPackageFragment()) - // It's ok to have hyphens in main file - usually no one imports it. - && (mainArtifact == null || !src.getLabel().equals(mainArtifact.getOwnerLabel()))) { - ruleContext.attributeError( - "srcs", - src.getLabel() - + ": paths to Python sources (besides the main source) may not contain '-'"); - } - Iterable pySrcs = - FileType.filter( - src.getProvider(FileProvider.class).getFilesToBuild().toList(), - PyRuleClasses.PYTHON_SOURCE); - Iterables.addAll(sourceFiles, pySrcs); - if (validate && Iterables.isEmpty(pySrcs)) { - ruleContext.attributeWarning( - "srcs", "rule '" + src.getLabel() + "' does not produce any Python source files"); - } - } - return sourceFiles; - } - - /** - * Returns true if any of this target's {@code deps} or {@code data} deps has a shared library - * file (e.g. a {@code .so}) in its transitive dependency closure. - * - *

For targets with the py provider, we consult the {@code uses_shared_libraries} field. For - * targets without this provider, we look for {@link CppFileTypes#SHARED_LIBRARY}-type files in - * the filesToBuild. - */ - private static boolean initUsesSharedLibraries(RuleContext ruleContext) { - Iterable targets; - // The deps attribute must exist for all rule types that use PyCommon, but not necessarily the - // data attribute. - if (ruleContext.attributes().has("data")) { - targets = - Iterables.concat( - ruleContext.getPrerequisites("deps"), ruleContext.getPrerequisites("data")); - } else { - targets = ruleContext.getPrerequisites("deps"); - } - for (TransitiveInfoCollection target : targets) { - if (target.get(PyInfo.PROVIDER) != null) { - if (target.get(PyInfo.PROVIDER).getUsesSharedLibraries()) { - return true; - } - } else if (FileType.contains( - target.getProvider(FileProvider.class).getFilesToBuild().toList(), - CppFileTypes.SHARED_LIBRARY)) { - return true; - } - } - return false; - } - - /** - * Returns the transitive import paths of a target. - * - *

For targets with the PyInfo provider, the value from that provider is used. Otherwise, we - * default to an empty set. - */ - private static NestedSet initImports(RuleContext ruleContext, PythonSemantics semantics) { - NestedSetBuilder builder = NestedSetBuilder.compileOrder(); - builder.addAll(semantics.getImports(ruleContext)); - for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps")) { - NestedSet imports; - if (dep.get(PyInfo.PROVIDER) != null) { - imports = dep.get(PyInfo.PROVIDER).getImportsSet(); - } else { - imports = NestedSetBuilder.emptySet(Order.COMPILE_ORDER); - } - if (!builder.getOrder().isCompatible(imports.getOrder())) { - // TODO(brandjon): We should make order an invariant of the Python provider, and move this - // check into PyInfo. - ruleContext.ruleError( - getOrderErrorMessage("imports", builder.getOrder(), imports.getOrder())); - } else { - builder.addTransitive(imports); - } - } - return builder.build(); - } - - /** - * Returns true if any of {@code deps} has a py provider with {@code has_py2_only_sources} set, or - * this target has a {@code srcs_version} of {@code PY2} or {@code PY2ONLY}. - */ - private static boolean initHasPy2OnlySources( - RuleContext ruleContext, PythonVersion sourcesVersion) { - if (sourcesVersion == PythonVersion.PY2 || sourcesVersion == PythonVersion.PY2ONLY) { - return true; - } - for (PyInfo depInfo : ruleContext.getPrerequisites("deps", PyInfo.PROVIDER)) { - if (depInfo.getHasPy2OnlySources()) { - return true; - } - } - return false; - } - - /** - * Returns true if any of {@code deps} has a py provider with {@code has_py3_only_sources} set, or - * this target has {@code srcs_version} of {@code PY3} or {@code PY3ONLY}. - */ - private static boolean initHasPy3OnlySources( - RuleContext ruleContext, PythonVersion sourcesVersion) { - if (sourcesVersion == PythonVersion.PY3 || sourcesVersion == PythonVersion.PY3ONLY) { - return true; - } - for (PyInfo depInfo : ruleContext.getPrerequisites("deps", PyInfo.PROVIDER)) { - if (depInfo.getHasPy3OnlySources()) { - return true; - } - } - return false; - } - - /** - * Retrieves the {@link PyRuntimeInfo} object in the given field of the given {@link - * ToolchainInfo}. - * - *

If the field holds {@code None}, null is returned instead. - * - *

If the field does not exist on the given {@code ToolchainInfo}, or is not a {@code - * PyRuntimeInfo} and not {@code None}, an error is reported on the {@code ruleContext} and null - * is returned. - * - *

If the {@code PyRuntimeInfo} does not have {@code expectedVersion} as its Python version, an - * error is reported on the {@code ruleContext} (but the provider is still returned). - */ - @Nullable - private static PyRuntimeInfo parseRuntimeField( - RuleContext ruleContext, - PythonVersion expectedVersion, - ToolchainInfo toolchainInfo, - String field) { - Object fieldValue; - try { - fieldValue = toolchainInfo.getValue(field); - } catch (EvalException e) { - ruleContext.ruleError( - String.format( - "Error parsing the Python toolchain's ToolchainInfo: Could not retrieve field " - + "'%s': %s", - field, e.getMessage())); - return null; - } - if (fieldValue == null) { - ruleContext.ruleError( - String.format( - "Error parsing the Python toolchain's ToolchainInfo: field '%s' is missing", field)); - return null; - } - if (fieldValue == NONE) { - return null; - } - if (!(fieldValue instanceof PyRuntimeInfo)) { - ruleContext.ruleError( - String.format( - "Error parsing the Python toolchain's ToolchainInfo: Expected a PyRuntimeInfo in " - + "field '%s', but got '%s'", - field, Starlark.type(fieldValue))); - return null; - } - PyRuntimeInfo pyRuntimeInfo = (PyRuntimeInfo) fieldValue; - if (pyRuntimeInfo.getPythonVersion() != expectedVersion) { - ruleContext.ruleError( - String.format( - "Error retrieving the Python runtime from the toolchain: Expected field '%s' to have " - + "a runtime with python_version = '%s', but got python_version = '%s'", - field, expectedVersion.name(), pyRuntimeInfo.getPythonVersion().name())); - } - return pyRuntimeInfo; - } - - /** - * Returns a {@link PyRuntimeInfo} representing the runtime to use for this target, as retrieved - * from the resolved Python toolchain. - * - *

If the configuration says to use the legacy mechanism for obtaining the runtime rather than - * the toolchain mechanism, OR if this target's rule class does not define the - * "$py_toolchain_type" attribute, then null is returned. In this case no attempt is made to - * retrieve any toolchain information, and no errors are reported. - * - *

Otherwise, the toolchain provider structure is retrieved and validated, and any errors are - * reported on the rule context. If we're unable to determine the runtime due to an error, or if - * the toolchain does not specify a runtime for the version of Python we need, null is returned. - * - * @throws IllegalArgumentException if the rule class defines the "$py_toolchain_type" attribute - * but does not declare a requirement on the toolchain type - */ - @Nullable - private static PyRuntimeInfo initRuntimeFromToolchain( - RuleContext ruleContext, PythonVersion version) { - if (!shouldGetRuntimeFromToolchain(ruleContext) - || !ruleContext.attributes().has("$py_toolchain_type", BuildType.NODEP_LABEL)) { - return null; - } - Label toolchainType = ruleContext.attributes().get("$py_toolchain_type", BuildType.NODEP_LABEL); - ToolchainInfo toolchainInfo = ruleContext.getToolchainInfo(toolchainType); - Preconditions.checkArgument( - toolchainInfo != null, - "Could not retrieve a Python toolchain for '%s' rule", - ruleContext.getRule().getRuleClass()); - - PyRuntimeInfo py2RuntimeInfo = - parseRuntimeField(ruleContext, PythonVersion.PY2, toolchainInfo, "py2_runtime"); - PyRuntimeInfo py3RuntimeInfo = - parseRuntimeField(ruleContext, PythonVersion.PY3, toolchainInfo, "py3_runtime"); - Preconditions.checkState(version == PythonVersion.PY2 || version == PythonVersion.PY3); - PyRuntimeInfo result = version == PythonVersion.PY2 ? py2RuntimeInfo : py3RuntimeInfo; - if (result == null) { - ruleContext.ruleError( - String.format( - "The Python toolchain does not provide a runtime for Python version %s", - version.name())); - } - - // Hack around the fact that the autodetecting Python toolchain, which is automatically - // registered, does not yet support windows. In this case, we want to return null so that - // BazelPythonSemantics falls back on --python_path. See toolchain.bzl. - // TODO(#7844): Remove this hack when the autodetecting toolchain has a windows implementation. - if (py2RuntimeInfo != null - && py2RuntimeInfo.getInterpreterPathString() != null - && py2RuntimeInfo - .getInterpreterPathString() - .equals("/_magic_pyruntime_sentinel_do_not_use")) { - return null; - } - - return result; - } - - /** - * Reports an attribute error if {@code python_version} cannot be parsed as {@code PY2}, {@code - * PY3}, or the sentinel value. - * - *

This *should* be enforced by rule attribute validation ({@link - * Attribute.Builder.allowedValues}), but this check is here to fail-fast just in case. - */ - private void validatePythonVersionAttr() { - AttributeMap attrs = ruleContext.attributes(); - if (!attrs.has(PYTHON_VERSION_ATTRIBUTE, Type.STRING)) { - return; - } - String attrValue = attrs.get(PYTHON_VERSION_ATTRIBUTE, Type.STRING); - try { - PythonVersion.parseTargetOrSentinelValue(attrValue); - } catch (IllegalArgumentException ex) { - ruleContext.attributeError( - PYTHON_VERSION_ATTRIBUTE, - String.format("'%s' is not a valid value. Expected either 'PY2' or 'PY3'", attrValue)); - } - } - - /** - * If the Python version (as determined by the configuration) is inconsistent with {@link - * #hasPy2OnlySources} or {@link #hasPy3OnlySources}, emits a {@link FailAction} that "generates" - * the executable. - * - *

We use a {@code FailAction} rather than a rule error because we want to defer the error - * until the execution phase. This way, we still get a configured target that the user can query - * over with an aspect to find the exact transitive dependency that introduced the offending - * version constraint. (See {@code //tools/python/srcs_version.bzl%find_requirements}) - * - * @return true if a {@link FailAction} was created - */ - private boolean maybeCreateFailActionDueToTransitiveSourcesVersion() { - String errorTemplate = - ruleContext.getLabel() - + ": " - + "This target is being built for Python %s but (transitively) includes Python %s-only " - + "sources. You can get diagnostic information about which dependencies introduce this " - + "version requirement by running the `find_requirements` aspect. For more info see " - + "the documentation for the `srcs_version` attribute: " - + semantics.getSrcsVersionDocURL(); - - String error = null; - if (version == PythonVersion.PY2 && hasPy3OnlySources) { - error = String.format(errorTemplate, "2", "3"); - } else if (version == PythonVersion.PY3 && hasPy2OnlySources) { - error = String.format(errorTemplate, "3", "2"); - } - if (error == null) { - return false; - } else { - ruleContext.registerAction( - new FailAction( - ruleContext.getActionOwner(), - ImmutableList.of(executable), - error, - Code.INCORRECT_PYTHON_VERSION)); - return true; - } - } - - public PythonVersion getVersion() { - return version; - } - - public PythonVersion getSourcesVersion() { - return sourcesVersion; - } - - /** - * Returns the transitive Python sources collected from the deps attribute, not including sources - * from the srcs attribute (unless they were separately reached via deps). - */ - public NestedSet getDependencyTransitivePythonSources() { - return dependencyTransitivePythonSources; - } - - /** Returns the transitive Python sources collected from the deps and srcs attributes. */ - public NestedSet getTransitivePythonSources() { - return transitivePythonSources; - } - - public boolean usesSharedLibraries() { - return usesSharedLibraries; - } - - public NestedSet getImports() { - return imports; - } - - public boolean hasPy2OnlySources() { - return hasPy2OnlySources; - } - - public boolean hasPy3OnlySources() { - return hasPy3OnlySources; - } - - /** - * Returns {@code true} if the Python runtime should be obtained from the Python toolchain (as per - * {@code --incompatible_use_python_toolchains}), as opposed to through the legacy mechanism - * specified in the {@link PythonSemantics} (e.g., {@code --python_top}). - */ - public boolean shouldGetRuntimeFromToolchain() { - return shouldGetRuntimeFromToolchain(ruleContext); - } - - private static boolean shouldGetRuntimeFromToolchain(RuleContext ruleContext) { - return ruleContext.getFragment(PythonConfiguration.class).useToolchains(); - } - - /** - * Returns a {@link PyRuntimeInfo} representing the runtime to use for this target, as retrieved - * from the resolved toolchain. - * - *

This may only be called for executable Python rules (rules defining the attribute - * "$py_toolchain_type", i.e. {@code py_binary} and {@code py_test}). In addition, it may not be - * called if {@link #shouldGetRuntimeFromToolchain()} returns false. - * - *

If there was a problem retrieving the runtime information from the toolchain, null is - * returned. An error would have already been reported on the rule context at {@code PyCommon} - * initialization time. - */ - @Nullable - public PyRuntimeInfo getRuntimeFromToolchain() { - Preconditions.checkArgument( - ruleContext.attributes().has("$py_toolchain_type", BuildType.NODEP_LABEL), - "Cannot retrieve Python toolchain information for '%s' rule", - ruleContext.getRule().getRuleClass()); - Preconditions.checkArgument( - shouldGetRuntimeFromToolchain(), - "Access to the Python toolchain is disabled by --incompatible_use_python_toolchains=false"); - - return runtimeFromToolchain; - } - - public void initBinary(List srcs) { - Preconditions.checkNotNull(version); - - if (OS.getCurrent() == OS.WINDOWS) { - executable = - ruleContext.getImplicitOutputArtifact(ruleContext.getTarget().getName() + ".exe"); - } else { - executable = ruleContext.createOutputArtifact(); - } - - NestedSetBuilder filesToBuildBuilder = - NestedSetBuilder.stableOrder().addAll(srcs).add(executable); - - if (ruleContext.getFragment(PythonConfiguration.class).buildPythonZip()) { - filesToBuildBuilder.add(getPythonZipArtifact(executable)); - } else if (OS.getCurrent() == OS.WINDOWS) { - // TODO(bazel-team): Here we should check target platform instead of using OS.getCurrent(). - // On Windows, add the python stub launcher in the set of files to build. - filesToBuildBuilder.add(getPythonStubArtifactForWindows(executable)); - } - - filesToBuild = filesToBuildBuilder.build(); - - if (ruleContext.hasErrors()) { - return; - } - - addPyExtraActionPseudoAction(); - } - - /** - * @return an artifact next to the executable file with a given suffix. - */ - private Artifact getArtifactWithExtension(Artifact executable, String extension) { - // On Windows, the Python executable has .exe extension on Windows, - // On Linux, the Python executable has no extension. - // We can't use ruleContext#getRelatedArtifact because it would mangle files with dots in the - // name on non-Windows platforms. - PathFragment pathFragment = - executable.getOutputDirRelativePath( - ruleContext.getConfiguration().isSiblingRepositoryLayout()); - String fileName = executable.getFilename(); - if (OS.getCurrent() == OS.WINDOWS) { - Preconditions.checkArgument(fileName.endsWith(".exe")); - fileName = fileName.substring(0, fileName.length() - 4) + extension; - } else { - fileName = fileName + extension; - } - return ruleContext.getDerivedArtifact(pathFragment.replaceName(fileName), executable.getRoot()); - } - - /** Returns an artifact next to the executable file with ".zip" suffix. */ - public Artifact getPythonZipArtifact(Artifact executable) { - return getArtifactWithExtension(executable, ".zip"); - } - - /** - * Returns an artifact next to the executable file with ".temp" suffix. Used only if we're - * building a zip. - */ - public Artifact getPythonIntermediateStubArtifact(Artifact executable) { - return getArtifactWithExtension(executable, ".temp"); - } - - /** Returns an artifact next to the executable file with no suffix. Only called for Windows. */ - public Artifact getPythonStubArtifactForWindows(Artifact executable) { - return ruleContext.getRelatedArtifact( - executable.getOutputDirRelativePath( - ruleContext.getConfiguration().isSiblingRepositoryLayout()), - ""); - } - - /** - * Adds a PyInfo provider. - * - *

This is a public method because some rules just want a PyInfo provider without the other - * things py_library needs. - */ - public void addPyInfoProvider(RuleConfiguredTargetBuilder builder) { - builder.addNativeDeclaredProvider( - PyInfo.builder() - .setTransitiveSources(transitivePythonSources) - .setUsesSharedLibraries(usesSharedLibraries) - .setImports(imports) - .setHasPy2OnlySources(hasPy2OnlySources) - .setHasPy3OnlySources(hasPy3OnlySources) - .build()); - } - - public void addCommonTransitiveInfoProviders(RuleConfiguredTargetBuilder builder) { - addPyInfoProvider(builder); - - // Add PyRuntimeInfo if this is an executable rule. - if (runtimeFromToolchain != null) { - builder.addNativeDeclaredProvider(runtimeFromToolchain); - } - - builder - .addNativeDeclaredProvider( - InstrumentedFilesCollector.collect(ruleContext, INSTRUMENTATION_SPEC)) - // Python targets are not really compilable. The best we can do is make sure that all - // generated source files are ready. - .addOutputGroup(OutputGroupInfo.FILES_TO_COMPILE, transitivePythonSources) - .addOutputGroup(OutputGroupInfo.COMPILATION_PREREQUISITES, transitivePythonSources); - } - - /** Returns a list of the source artifacts */ - public List getPythonSources() { - return directPythonSources; - } - - /** - * Adds a {@link PseudoAction} to the build graph that is only used for providing information to - * the blaze extra_action feature. - */ - void addPyExtraActionPseudoAction() { - if (ruleContext.getConfiguration().getActionListeners().isEmpty()) { - return; - } - registerPyExtraActionPseudoAction(ruleContext, dependencyTransitivePythonSources); - } + private PyCommon() {} // Public so that Starlark bindings can access it. Should only be called by PyStarlarkBuiltins. // TODO(b/253059598): Remove support for this; https://github.com/bazelbuild/bazel/issues/16455 @@ -788,7 +80,7 @@ public static void registerPyExtraActionPseudoAction( * Creates a {@link PseudoAction} that is only used for providing information to the blaze * extra_action feature. */ - public static Action makePyExtraActionPseudoAction( + private static Action makePyExtraActionPseudoAction( ActionOwner owner, List sources, NestedSet dependencies, @@ -815,121 +107,6 @@ public static Action makePyExtraActionPseudoAction( @SerializationConstant @AutoCodec.VisibleForSerialization static final GeneratedExtension PYTHON_INFO = PythonInfo.pythonInfo; - /** Returns the artifact for the main Python source file. */ - private static Artifact initMainArtifact(RuleContext ruleContext) { - String mainSourceName; - Rule target = ruleContext.getRule(); - boolean explicitMain = target.isAttributeValueExplicitlySpecified("main"); - if (explicitMain) { - mainSourceName = ruleContext.attributes().get("main", BuildType.LABEL).getName(); - if (!mainSourceName.endsWith(".py")) { - ruleContext.attributeError("main", "main must end in '.py'"); - } - } else { - String ruleName = target.getName(); - // TODO(blaze-team) Shouldn't the same check apply for all Python targets? - if (ruleName.endsWith(".py")) { - ruleContext.attributeError("name", "name must not end in '.py'"); - } - mainSourceName = ruleName + ".py"; - } - PathFragment mainSourcePath = PathFragment.create(mainSourceName); - - Artifact mainArtifact = null; - for (Artifact outItem : ruleContext.getPrerequisiteArtifacts("srcs").list()) { - if (outItem.getRootRelativePath().endsWith(mainSourcePath)) { - if (mainArtifact == null) { - mainArtifact = outItem; - } else { - ruleContext.attributeError( - "srcs", - buildMultipleMainMatchesErrorText( - explicitMain, - mainSourceName, - mainArtifact.getRunfilesPath().toString(), - outItem.getRunfilesPath().toString())); - } - } - } - - if (mainArtifact == null) { - ruleContext.attributeError("srcs", buildNoMainMatchesErrorText(explicitMain, mainSourceName)); - } - - return mainArtifact; - } - - /** - * Returns the path to the main python entry point, or null if this PyCommon was initialized with - * {@code requiresMainFile} set to false (or there was an error in determining the main artifact). - */ - @Nullable - public String determineMainExecutableSource() { - if (mainArtifact == null) { - return null; - } - PathFragment workspaceName = - PathFragment.create(ruleContext.getRule().getPackage().getWorkspaceName()); - return workspaceName.getRelative(mainArtifact.getRunfilesPath()).getPathString(); - } - - public Artifact getExecutable() { - return executable; - } - - public NestedSet getFilesToBuild() { - return filesToBuild; - } - - /** - * Creates the actual executable artifact, i.e., emits a generating action for {@link - * #getExecutable()}. - * - *

If there is a transitive sources version conflict, may produce a {@link FailAction} to - * trigger an execution-time failure. See {@link - * #maybeCreateFailActionDueToTransitiveSourcesVersion}. - */ - public void createExecutable(CcInfo ccInfo, Runfiles.Builder defaultRunfilesBuilder) - throws InterruptedException, RuleErrorException { - boolean failed = maybeCreateFailActionDueToTransitiveSourcesVersion(); - if (!failed) { - semantics.createExecutable(ruleContext, this, ccInfo, defaultRunfilesBuilder); - } - } - - private static String buildMultipleMainMatchesErrorText( - boolean explicit, String proposedMainName, String match1, String match2) { - String errorText; - if (explicit) { - errorText = - String.format( - "file name '%s' specified by 'main' attribute matches multiple files: e.g., '%s' and" - + " '%s'", - proposedMainName, match1, match2); - } else { - errorText = - String.format( - "default main file name '%s' matches multiple files. Perhaps specify an explicit" - + " file with 'main' attribute? Matches were: '%s' and '%s'", - proposedMainName, match1, match2); - } - return errorText; - } - - private static String buildNoMainMatchesErrorText(boolean explicit, String proposedMainName) { - String errorText; - if (explicit) { - errorText = "could not find '" + proposedMainName + "' as specified by 'main' attribute"; - } else { - errorText = - String.format( - "corresponding default '%s' does not appear in srcs. Add it or override default file" - + " name with a 'main' attribute", - proposedMainName); - } - return errorText; - } - // Used purely to set the legacy ActionType of the ExtraActionInfo. @Immutable private static final class PyPseudoAction extends PseudoAction {