Skip to content

Commit

Permalink
Remove errant paths from crosstool test files.
Browse files Browse the repository at this point in the history
Also includes the following changes:

Introduce a helper class for low-boilerplate time measurements and logging.

--
Removed attribute "args" from java_test.

--
Adds the arm64, mips, mips64, x86, and x86_64 toolchains in the Android NDK

This adds the precited targets to the crosstools file generated by the android_ndk_repository rule. The crosstools support NDK revision r10e-rc4.

RELNOTES: arm64, mips, mips64, x86, and x86_64 NDK toolchains added to android_ndk_repository in Bazel

--
MOS_MIGRATED_REVID=100953441
  • Loading branch information
ahumesky authored and lberki committed Aug 20, 2015
1 parent 28fe4da commit a2e2cc2
Show file tree
Hide file tree
Showing 20 changed files with 54,318 additions and 240 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,33 +13,43 @@
// limitations under the License.
package com.google.devtools.build.lib.bazel.rules.android;

import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.bazel.repository.RepositoryFunction;
import com.google.devtools.build.lib.bazel.rules.android.ndkcrosstools.AndroidNdkCrosstools;
import com.google.devtools.build.lib.bazel.rules.android.ndkcrosstools.AndroidNdkCrosstools.NdkCrosstoolsException;
import com.google.devtools.build.lib.bazel.rules.android.ndkcrosstools.NdkPaths;
import com.google.devtools.build.lib.bazel.rules.android.ndkcrosstools.NdkRelease;
import com.google.devtools.build.lib.packages.AttributeMap;
import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper;
import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.Type;
import com.google.devtools.build.lib.skyframe.FileSymlinkException;
import com.google.devtools.build.lib.skyframe.FileValue;
import com.google.devtools.build.lib.util.CPU;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.skyframe.InconsistentFilesystemException;
import com.google.devtools.build.lib.util.ResourceFileLoader;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain;
import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CrosstoolRelease;
import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.DefaultCpuToolchain;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
* Implementation of the {@code android_ndk_repository} rule.
*/
public class AndroidNdkRepositoryFunction extends RepositoryFunction {

@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
RepositoryName repositoryName = (RepositoryName) skyKey.argument();
Expand Down Expand Up @@ -69,80 +79,16 @@ ndkSymlinkTreeDirectory, getOutputBase().getFileSystem().getPath(pathFragment),
AttributeMap attributes = NonconfigurableAttributeMapper.of(rule);
String ruleName = rule.getName();
String apiLevel = attributes.get("api_level", Type.INTEGER).toString();
List<String> cpus = ImmutableList.of("arm"); // TODO(bazel-team): autodetect
String abi = "armeabi-v7a"; // TODO(bazel-team): Should this be an attribute on the rule?
String compiler = "4.9"; // TODO(bazel-team): Should this be an attribute on the rule?

// TODO(bazel-team): move this, add other cases
String hostPlatform, hostArch;
switch (OS.getCurrent()) {
case DARWIN:
hostPlatform = "darwin";
break;
case LINUX:
hostPlatform = "linux";
break;
default:
hostPlatform = "unknown";
}
switch (CPU.getCurrent()) {
case X86_32:
hostArch = "x86";
break;
case X86_64:
hostArch = "x86_64";
break;
default:
hostArch = "unknown";
}

String hostCpu = hostPlatform + "-" + hostArch;

String ccToolchainSuiteTemplate;
String ccToolchainTemplate;
String toolchainTemplate;

CrosstoolRelease androidCrosstoolRelease;
try {
ccToolchainSuiteTemplate = ResourceFileLoader.loadResource(
AndroidNdkRepositoryFunction.class, "android_ndk_cc_toolchain_suite_template.txt");
ccToolchainTemplate = ResourceFileLoader.loadResource(
AndroidNdkRepositoryFunction.class, "android_ndk_cc_toolchain_template.txt");
toolchainTemplate = ResourceFileLoader.loadResource(
AndroidNdkRepositoryFunction.class, "android_ndk_toolchain_template.txt");
} catch (IOException e) {
throw new IllegalStateException(e);
}

StringBuilder toolchainMap = new StringBuilder();
StringBuilder toolchainProtos = new StringBuilder();
StringBuilder toolchains = new StringBuilder();

for (String cpu : cpus) {
toolchainMap.append(String.format("\"%s\": \":cc-compiler-%s\", ", abi, abi));
toolchainProtos.append(
toolchainTemplate
.replace("%repository%", ruleName)
.replace("%host_cpu%", hostCpu)
.replace("%cpu%", cpu)
.replace("%abi%", abi)
.replace("%api_level%", apiLevel)
.replace("%compiler%", compiler));
toolchains.append(
ccToolchainTemplate
.replace("%repository%", ruleName)
.replace("%host_cpu%", hostCpu)
.replace("%cpu%", cpu)
.replace("%abi%", abi)
.replace("%api_level%", apiLevel)
.replace("%compiler%", compiler));
androidCrosstoolRelease = AndroidNdkCrosstools.createCrosstoolRelease(
env.getListener(), ruleName, apiLevel, getNdkRelease(directoryValue, env));
} catch (NdkCrosstoolsException e) {
throw new RepositoryFunctionException(new IOException(e), Transience.PERSISTENT);
}

String buildFile = ccToolchainSuiteTemplate
.replace("%toolchain_map%", toolchainMap)
.replace("%toolchain_protos%", toolchainProtos)
.replace("%toolchains%", toolchains)
.replace("%default_cpu%", abi);

String buildFile = createBuildFile(androidCrosstoolRelease, ruleName);
return writeBuildFile(directoryValue, buildFile);
}

Expand All @@ -155,4 +101,103 @@ public SkyFunctionName getSkyFunctionName() {
public Class<? extends RuleDefinition> getRuleDefinition() {
return AndroidNdkRepositoryRule.class;
}

private static String createBuildFile(CrosstoolRelease androidCrosstoolRelease, String ruleName) {

String ccToolchainSuiteTemplate = getTemplate("android_ndk_cc_toolchain_suite_template.txt");
String ccToolchainTemplate = getTemplate("android_ndk_cc_toolchain_template.txt");

StringBuilder toolchainMap = new StringBuilder();
for (DefaultCpuToolchain defaultToolchain : androidCrosstoolRelease.getDefaultToolchainList()) {
toolchainMap.append(String.format(" \"%s\": \":%s\",\n",
defaultToolchain.getCpu(), defaultToolchain.getToolchainIdentifier()));
}

StringBuilder ccToolchainRules = new StringBuilder();
for (CToolchain toolchain : androidCrosstoolRelease.getToolchainList()) {
ccToolchainRules.append(createCcToolchainRule(ccToolchainTemplate, toolchain));
}

String crosstoolReleaseProto = androidCrosstoolRelease.toString();

return ccToolchainSuiteTemplate
.replace("%ruleName%", ruleName)
.replace("%toolchainMap%", toolchainMap.toString().trim())
.replace("%crosstoolReleaseProto%", crosstoolReleaseProto)
.replace("%ccToolchainRules%", ccToolchainRules);
}

private static String createCcToolchainRule(String ccToolchainTemplate, CToolchain toolchain) {

// TODO(bazel-team): It's unfortunate to have to extract data from a CToolchain proto like this.
// It would be better to have a higher-level construction (like an AndroidToolchain class)
// from which the CToolchain proto and rule information needed here can be created.
// Alternatively it would be nicer to just be able to glob the entire NDK and add that one glob
// to each cc_toolchain rule, and then the complexities in the method and the templates can
// go away, but globbing the entire NDK takes ~60 seconds, mostly because of MD5ing all the
// binary files in the NDK (eg the .so / .a / .o files).

// This also includes the files captured with cxx_builtin_include_directory
String toolchainDirectory = NdkPaths.getToolchainDirectoryFromToolPath(
toolchain.getToolPathList().get(0).getPath());

// Create file glob patterns for the various files that the toolchain references.

String androidPlatformIncludes = NdkPaths.stripRepositoryPrefix(toolchain.getBuiltinSysroot())
+ "/**/*";

List<String> toolchainFileGlobPatterns = new ArrayList<String>();
toolchainFileGlobPatterns.add(androidPlatformIncludes);

for (String cxxFlag : toolchain.getUnfilteredCxxFlagList()) {
if (!cxxFlag.startsWith("-")) { // Skip flag names
toolchainFileGlobPatterns.add(NdkPaths.stripRepositoryPrefix(cxxFlag) + "/**/*");
}
}

StringBuilder toolchainFileGlobs = new StringBuilder();
for (String toolchainFileGlobPattern : toolchainFileGlobPatterns) {
toolchainFileGlobs.append(String.format(
" \"%s\",\n", toolchainFileGlobPattern));
}

return ccToolchainTemplate
.replace("%toolchainName%", toolchain.getToolchainIdentifier())
.replace("%cpu%", toolchain.getTargetCpu())
.replace("%toolchainDirectory%", toolchainDirectory)
.replace("%toolchainFileGlobs%", toolchainFileGlobs.toString().trim());
}

private static NdkRelease getNdkRelease(FileValue directoryValue, Environment env)
throws RepositoryFunctionException {

Path releaseFilePath = directoryValue.realRootedPath().asPath().getRelative("ndk/RELEASE.TXT");

SkyKey releaseFileKey = FileValue.key(RootedPath.toRootedPath(
releaseFilePath, PathFragment.EMPTY_FRAGMENT));

String releaseFileContent;
try {
env.getValueOrThrow(releaseFileKey,
IOException.class,
FileSymlinkException.class,
InconsistentFilesystemException.class);

releaseFileContent = new String(FileSystemUtils.readContent(releaseFilePath));
} catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
throw new RepositoryFunctionException(
new IOException("Could not read RELEASE.TXT in Android NDK: " + e.getMessage()),
Transience.PERSISTENT);
}

return NdkRelease.create(releaseFileContent.trim());
}

private static String getTemplate(String templateFile) {
try {
return ResourceFileLoader.loadResource(AndroidNdkRepositoryFunction.class, templateFile);
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,18 +1,34 @@
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This build file was automatically generated for the
# android_ndk_repository rule "%ruleName%"

package(default_visibility = ["//visibility:public"])

cc_toolchain_suite(
name = "toolchain",
toolchains = { %toolchain_map% },
toolchains = {
%toolchainMap%
},
proto = """
major_version: "android"
minor_version: ""
default_target_cpu: "%default_cpu%"
%toolchain_protos%
%crosstoolReleaseProto%
""")

cc_library(
name = "malloc",
srcs = [],
)

%toolchains%
%ccToolchainRules%
Original file line number Diff line number Diff line change
@@ -1,41 +1,32 @@
filegroup(
name = "%abi%-android-%compiler%-toolchain",
srcs = glob([
"toolchains/%cpu%-linux-androideabi-%compiler%/**",
]),
output_licenses = ["unencumbered"],
)

filegroup(
name = "%abi%-android-%api_level%-%compiler%-files",
srcs = [
":everything-%api_level%-%abi%",
":%abi%-android-%compiler%-toolchain",
],
)
################################################################
# %toolchainName%
################################################################

cc_toolchain(
name = "cc-compiler-%abi%",
all_files = ":%abi%-android-%api_level%-%compiler%-files",
compiler_files = ":%abi%-android-%compiler%-toolchain",
cpu = "%abi%",
dwp_files = ":%abi%-android-%compiler%-toolchain",
dynamic_runtime_libs = [":%abi%-android-%compiler%-toolchain"],
linker_files = ":%abi%-android-%compiler%-toolchain",
objcopy_files = ":%abi%-android-%compiler%-toolchain",
static_runtime_libs = [":%abi%-android-%compiler%-toolchain"],
strip_files = ":%abi%-android-%compiler%-toolchain",
name = "%toolchainName%",
all_files = ":%toolchainName%-all_files",
compiler_files = ":%toolchainName%-toolchain_files",
cpu = "%cpu%",
dwp_files = ":%toolchainName%-toolchain_files",
dynamic_runtime_libs = [":%toolchainName%-toolchain_files"],
linker_files = ":%toolchainName%-toolchain_files",
objcopy_files = ":%toolchainName%-toolchain_files",
static_runtime_libs = [":%toolchainName%-toolchain_files"],
strip_files = ":%toolchainName%-toolchain_files",
supports_param_files = 0,
)

filegroup(
name = "everything-%api_level%-%abi%",
srcs = glob(
[
"ndk/platforms/android-%api_level%/arch-%cpu%/usr/include/**/*.h",
"ndk/platforms/android-%api_level%/arch-%cpu%/usr/lib/**/*.a",
"ndk/platforms/android-%api_level%/arch-%cpu%/usr/lib/**/*.o",
"ndk/platforms/android-%api_level%/arch-%cpu%/usr/lib/**/*.so",
],
),
name = "%toolchainName%-toolchain_files",
srcs = glob(["toolchains/%toolchainDirectory%/**"]),
output_licenses = ["unencumbered"],
)

filegroup(
name = "%toolchainName%-all_files",
srcs = [
":%toolchainName%-toolchain_files",
] + glob([
%toolchainFileGlobs%
]),
)
Loading

0 comments on commit a2e2cc2

Please sign in to comment.