Skip to content

Commit

Permalink
Building with vendored repos in offline mode
Browse files Browse the repository at this point in the history 
Update the build logic in vendor mode when `--nofetch` is passed:

If repo exists in the vendor directory
- If repo up-to-date ---> build
- else ---> warn that out-of-date version is used & build

else ---> fail

Working towards: #19563

PiperOrigin-RevId: 604405285
Change-Id: Ic6c7807296d039e1bd8970c3cde57e68f5e8de86
  • Loading branch information
@SalmaSamy
SalmaSamy committed Feb 13, 2024
1 parent 86a548f commit b3f811d
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 30 deletions.
88 changes: 63 additions & 25 deletions ...main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,6 @@ public SkyValue compute(SkyKey skyKey, Environment env)
if (env.valuesMissing()) {
return null;
}

if (rule == null) {
return new NoRepositoryDirectoryValue(
String.format("Repository '%s' is not defined", repositoryName.getCanonicalForm()));
Expand All @@ -168,41 +167,23 @@ public SkyValue compute(SkyKey skyKey, Environment env)

DigestWriter digestWriter = new DigestWriter(directories, repositoryName, rule);
if (shouldUseVendorRepos(env, handler, rule)) {
Path vendorPath = VENDOR_DIRECTORY.get(env).get();
Path vendorMarker = vendorPath.getChild("@" + repositoryName.getName() + ".marker");
boolean isVendorRepoUpToDate =
digestWriter.areRepositoryAndMarkerFileConsistent(handler, env, vendorMarker) != null;
RepositoryDirectoryValue repositoryDirectoryValue =
tryGettingValueUsingVendoredRepo(
env, rule, repoRoot, repositoryName, handler, digestWriter);
if (env.valuesMissing()) {
return null;
}
if (isVendorRepoUpToDate) {
PathFragment vendorRepoPath =
vendorPath.getRelative(repositoryName.getName()).asFragment();
return setupOverride(vendorRepoPath, env, repoRoot, repositoryName.getName());
} else if (!IS_VENDOR_COMMAND.get(env).booleanValue()) {
// If this is vendor command, proceed with fetching to update the vendor directory.
// Otherwise, for example, the build command, we need to display a warning indicating
// that the vendored repository is out-of-date.
env.getListener()
.handle(
Event.warn(
rule.getLocation(),
String.format(
"Vendored repository '%s' is out-of-date. The up-to-date version will"
+ " be fetched into the external cache and used. To update the repo"
+ " in the vendor directory, run 'bazel vendor' with the directory"
+ " flag",
rule.getName())));
if (repositoryDirectoryValue != null) {
return repositoryDirectoryValue;
}
}

if (shouldUseCachedRepos(env, handler, repoRoot, rule)) {
// Make sure marker file is up-to-date; correctly describes the current repository state
byte[] markerHash = digestWriter.areRepositoryAndMarkerFileConsistent(handler, env);
if (env.valuesMissing()) {
return null;
}
if (markerHash != null) {
if (markerHash != null) { // repo exist & up-to-date
return RepositoryDirectoryValue.builder()
.setPath(repoRoot)
.setDigest(markerHash)
Expand Down Expand Up @@ -268,6 +249,63 @@ public SkyValue compute(SkyKey skyKey, Environment env)
}
}

@Nullable
private RepositoryDirectoryValue tryGettingValueUsingVendoredRepo(
Environment env,
Rule rule,
Path repoRoot,
RepositoryName repositoryName,
RepositoryFunction handler,
DigestWriter digestWriter)
throws RepositoryFunctionException, InterruptedException {
Path vendorPath = VENDOR_DIRECTORY.get(env).get();
Path vendorRepoPath = vendorPath.getRelative(repositoryName.getName());
if (vendorRepoPath.exists()) {
Path vendorMarker = vendorPath.getChild("@" + repositoryName.getName() + ".marker");
boolean isVendorRepoUpToDate =
digestWriter.areRepositoryAndMarkerFileConsistent(handler, env, vendorMarker) != null;
if (env.valuesMissing()) {
return null;
}
// If our repo is up-to-date, or this is an offline build (--nofetch), then the vendored repo
// is used.
if (isVendorRepoUpToDate || (!IS_VENDOR_COMMAND.get(env).booleanValue() && !isFetch.get())) {
if (!isVendorRepoUpToDate) { // If the repo is out-of-date, show a warning
env.getListener()
.handle(
Event.warn(
rule.getLocation(),
String.format(
"Vendored repository '%s' is out-of-date and fetching is disabled."
+ " Run build without the '--nofetch' option or run"
+ " `bazel vendor` to update it",
rule.getName())));
}
return setupOverride(vendorRepoPath.asFragment(), env, repoRoot, repositoryName.getName());
} else if (!IS_VENDOR_COMMAND.get(env).booleanValue()) { // build command & fetch enabled
// We will continue fetching but warn the user that we are not using the vendored repo
env.getListener()
.handle(
Event.warn(
rule.getLocation(),
String.format(
"Vendored repository '%s' is out-of-date. The up-to-date version will"
+ " be fetched into the external cache and used. To update the repo"
+ " in the vendor directory, run 'bazel vendor'",
rule.getName())));
}
} else if (!isFetch.get()) { // repo not vendored & fetching is disabled (--nofetch)
throw new RepositoryFunctionException(
new IOException(
"Vendored repository "
+ repositoryName.getName()
+ " not found under the vendor directory and fetching is disabled."
+ " To fix run 'bazel vendor' or build without the '--nofetch'"),
Transience.TRANSIENT);
}
return null;
}

@Nullable
private Rule getRepositoryRule(
Environment env, RepositoryName repositoryName, StarlarkSemantics starlarkSemantics)
Expand Down
95 changes: 90 additions & 5 deletions src/test/py/bazel/bzlmod/bazel_vendor_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,12 @@ def testBuildingWithVendoredRepos(self):

# Empty external & build with vendor
self.RunBazel(['clean', '--expunge'])
self.RunBazel(['build', '@aaa//:all', '--vendor_dir=vendor'])
_, _, stderr = self.RunBazel(['build', '@aaa//:all', '--vendor_dir=vendor'])
self.assertNotIn(
"Vendored repository '_main~ext~justRepo' is out-of-date.",
'\n'.join(stderr),
)

# Assert repo aaa in {OUTPUT_BASE}/external is a symlink (junction on
# windows, this validates it was created from vendor and not fetched)=
_, stdout, _ = self.RunBazel(['info', 'output_base'])
Expand Down Expand Up @@ -250,7 +255,7 @@ def testIgnoreFromVendoring(self):
self.assertNotIn('_main~ext~localRepo', repos_vendored)
self.assertNotIn('_main~ext~configRepo', repos_vendored)

def testOutOfDateVendoredRepo(self):
def testBuildingOutOfDateVendoredRepo(self):
self.ScratchFile(
'MODULE.bazel',
[
Expand Down Expand Up @@ -283,7 +288,7 @@ def testOutOfDateVendoredRepo(self):
"WARNING: <builtin>: Vendored repository '_main~ext~justRepo' is"
' out-of-date. The up-to-date version will be fetched into the external'
' cache and used. To update the repo in the vendor directory, run'
" 'bazel vendor' with the directory flag",
" 'bazel vendor'",
stderr,
)

Expand Down Expand Up @@ -313,7 +318,7 @@ def testOutOfDateVendoredRepo(self):
"WARNING: <builtin>: Vendored repository '_main~ext~justRepo' is"
' out-of-date. The up-to-date version will be fetched into the external'
' cache and used. To update the repo in the vendor directory, run'
" 'bazel vendor' with the directory flag",
" 'bazel vendor'",
stderr,
)
_, stdout, _ = self.RunBazel(['info', 'output_base'])
Expand All @@ -329,7 +334,87 @@ def testOutOfDateVendoredRepo(self):
"WARNING: <builtin>: Vendored repository '_main~ext~justRepo' is"
' out-of-date. The up-to-date version will be fetched into the external'
' cache and used. To update the repo in the vendor directory, run'
" 'bazel vendor' with the directory flag",
" 'bazel vendor'",
stderr,
)

def testBuildingVendoredRepoInOfflineMode(self):
self.ScratchFile(
'MODULE.bazel',
[
'ext = use_extension("extension.bzl", "ext")',
'use_repo(ext, "venRepo")',
],
)
self.ScratchFile(
'extension.bzl',
[
'def _repo_rule_impl(ctx):',
' ctx.file("WORKSPACE")',
' ctx.file("BUILD", "filegroup(name=\'lala\')")',
'repo_rule = repository_rule(implementation=_repo_rule_impl)',
'',
'def _ext_impl(ctx):',
' repo_rule(name="venRepo")',
'ext = module_extension(implementation=_ext_impl)',
],
)
self.ScratchFile('BUILD')

# Vendor, assert and build with no problems
self.RunBazel(['vendor', '--vendor_dir=vendor'])
self.assertIn('_main~ext~venRepo', os.listdir(self._test_cwd + '/vendor'))

# Make updates in repo definition
self.ScratchFile(
'MODULE.bazel',
[
'ext = use_extension("extension.bzl", "ext")',
'use_repo(ext, "venRepo")',
'use_repo(ext, "noVenRepo")',
],
)
self.ScratchFile(
'extension.bzl',
[
'def _repo_rule_impl(ctx):',
' ctx.file("WORKSPACE")',
' ctx.file("BUILD", "filegroup(name=\'haha\')")',
'repo_rule = repository_rule(implementation=_repo_rule_impl)',
'',
'def _ext_impl(ctx):',
' repo_rule(name="venRepo")',
' repo_rule(name="noVenRepo")',
'ext = module_extension(implementation=_ext_impl)',
],
)

# Building a repo that is not vendored in offline mode, should fail
_, _, stderr = self.RunBazel(
['build', '@noVenRepo//:all', '--vendor_dir=vendor', '--nofetch'],
allow_failure=True,
)
self.assertIn(
'ERROR: Vendored repository _main~ext~noVenRepo not found under the'
" vendor directory and fetching is disabled. To fix run 'bazel"
" vendor' or build without the '--nofetch'",
stderr,
)

# Building out-of-date repo in offline mode, should build the out-dated one
# and emit a warning
_, _, stderr = self.RunBazel(
['build', '@venRepo//:all', '--vendor_dir=vendor', '--nofetch'],
)
self.assertIn(
"WARNING: <builtin>: Vendored repository '_main~ext~venRepo' is"
' out-of-date and fetching is disabled. Run build without the'
" '--nofetch' option or run `bazel vendor` to update it",
stderr,
)
# Assert the out-dated repo is the one built with
self.assertIn(
'Target @@_main~ext~venRepo//:lala up-to-date (nothing to build)',
stderr,
)

Expand Down

0 comments on commit b3f811d

Please sign in to comment.