Skip to content

Commit b2bdf97

Browse files
SalmaSamycopybara-github
SalmaSamy
authored and
copybara-github
committed
Vendor Target
Use the TargetFetcher used for fetch to fetch targets, while internally runs a build and creates a dependency graph. Then traverse this graph for each target to collect the repos and vendor them. PiperOrigin-RevId: 627047679 Change-Id: I78a33c4258099a45d4d3a345bd075badc8985209
1 parent 4d966a0 commit b2bdf97

8 files changed

Lines changed: 189 additions & 23 deletions

File tree

‎src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ java_library(
5353
"//src/main/java/com/google/devtools/build/lib/rules:repository/resolved_file_value",
5454
"//src/main/java/com/google/devtools/build/lib/runtime/commands",
5555
"//src/main/java/com/google/devtools/build/lib/shell",
56+
"//src/main/java/com/google/devtools/build/lib/skyframe:configured_target_key",
5657
"//src/main/java/com/google/devtools/build/lib/skyframe:package_lookup_value",
5758
"//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
5859
"//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value",

‎src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,13 +217,15 @@ private BlazeCommandResult fetchRepos(
217217
private BlazeCommandResult fetchTarget(
218218
CommandEnvironment env, OptionsParsingResult options, List<String> targets) {
219219
try {
220-
TargetFetcher.fetchTargets(env, options, targets);
220+
var unused = TargetFetcher.fetchTargets(env, options, targets);
221221
} catch (TargetFetcherException e) {
222222
return createFailedBlazeCommandResult(
223223
env.getReporter(), Code.QUERY_EVALUATION_ERROR, e.getMessage());
224224
}
225225
env.getReporter()
226-
.handle(Event.info("All external dependencies for these targets fetched successfully."));
226+
.handle(
227+
Event.info(
228+
"All external dependencies for the requested targets fetched successfully."));
227229
return BlazeCommandResult.success();
228230
}
229231

‎src/main/java/com/google/devtools/build/lib/bazel/commands/TargetFetcher.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,13 @@ private TargetFetcher(CommandEnvironment env) {
3434
}
3535

3636
/** Creates a no-build build request to fetch all repos needed to build these targets */
37-
public static void fetchTargets(
37+
public static BuildResult fetchTargets(
3838
CommandEnvironment env, OptionsParsingResult options, List<String> targets)
3939
throws TargetFetcherException {
40-
new TargetFetcher(env).fetchTargets(options, targets);
40+
return new TargetFetcher(env).fetchTargets(options, targets);
4141
}
4242

43-
private void fetchTargets(OptionsParsingResult options, List<String> targets)
43+
private BuildResult fetchTargets(OptionsParsingResult options, List<String> targets)
4444
throws TargetFetcherException {
4545
BuildRequest request =
4646
BuildRequest.builder()
@@ -59,6 +59,7 @@ private void fetchTargets(OptionsParsingResult options, List<String> targets)
5959
"Fetching some target dependencies failed with errors: "
6060
+ result.getDetailedExitCode().getFailureDetail().getMessage());
6161
}
62+
return result;
6263
}
6364

6465
static void injectNoBuildOption(OptionsParser optionsParser) {

‎src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java‎

Lines changed: 96 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@
2323
import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent;
2424
import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllValue;
2525
import com.google.devtools.build.lib.bazel.commands.RepositoryFetcher.RepositoryFetcherException;
26+
import com.google.devtools.build.lib.bazel.commands.TargetFetcher.TargetFetcherException;
2627
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
28+
import com.google.devtools.build.lib.buildtool.BuildResult;
2729
import com.google.devtools.build.lib.cmdline.LabelConstants;
2830
import com.google.devtools.build.lib.cmdline.RepositoryName;
2931
import com.google.devtools.build.lib.events.Event;
@@ -38,12 +40,13 @@
3840
import com.google.devtools.build.lib.runtime.CommandEnvironment;
3941
import com.google.devtools.build.lib.runtime.KeepGoingOption;
4042
import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption;
43+
import com.google.devtools.build.lib.runtime.commands.TestCommand;
4144
import com.google.devtools.build.lib.server.FailureDetails;
4245
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
4346
import com.google.devtools.build.lib.server.FailureDetails.FetchCommand.Code;
47+
import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
4448
import com.google.devtools.build.lib.skyframe.PrecomputedValue;
4549
import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException;
46-
import com.google.devtools.build.lib.util.AbruptExitException;
4750
import com.google.devtools.build.lib.util.DetailedExitCode;
4851
import com.google.devtools.build.lib.util.InterruptedFailureDetails;
4952
import com.google.devtools.build.lib.vfs.FileSystemUtils;
@@ -52,25 +55,37 @@
5255
import com.google.devtools.build.lib.vfs.Symlinks;
5356
import com.google.devtools.build.skyframe.EvaluationContext;
5457
import com.google.devtools.build.skyframe.EvaluationResult;
58+
import com.google.devtools.build.skyframe.InMemoryGraph;
59+
import com.google.devtools.build.skyframe.NodeEntry;
60+
import com.google.devtools.build.skyframe.QueryableGraph.Reason;
5561
import com.google.devtools.build.skyframe.SkyKey;
5662
import com.google.devtools.build.skyframe.SkyValue;
63+
import com.google.devtools.common.options.OptionsParser;
5764
import com.google.devtools.common.options.OptionsParsingResult;
5865
import java.io.IOException;
66+
import java.util.ArrayDeque;
5967
import java.util.ArrayList;
68+
import java.util.HashSet;
6069
import java.util.List;
6170
import java.util.Map.Entry;
6271
import java.util.Objects;
72+
import java.util.Queue;
73+
import java.util.Set;
6374
import javax.annotation.Nullable;
6475

6576
/** Fetches external repositories into a specified directory. */
6677
@Command(
6778
name = VendorCommand.NAME,
79+
builds = true,
80+
inherits = {TestCommand.class},
6881
options = {
6982
VendorOptions.class,
7083
PackageOptions.class,
7184
KeepGoingOption.class,
7285
LoadingPhaseThreadsOption.class
7386
},
87+
allowResidue = true,
88+
usesConfigurationOptions = true,
7489
help = "resource:vendor.txt",
7590
shortDescription =
7691
"Fetches external repositories into a specific folder specified by the flag "
@@ -81,6 +96,14 @@ public final class VendorCommand implements BlazeCommand {
8196
// TODO(salmasamy) decide on name and format
8297
private static final String VENDOR_IGNORE = ".vendorignore";
8398

99+
@Override
100+
public void editOptions(OptionsParser optionsParser) {
101+
// We only need to inject these options with fetch target (when there is a residue)
102+
if (!optionsParser.getResidue().isEmpty()) {
103+
TargetFetcher.injectNoBuildOption(optionsParser);
104+
}
105+
}
106+
84107
@Override
85108
public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) {
86109
BlazeCommandResult invalidResult = validateOptions(env, options);
@@ -109,15 +132,13 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti
109132
PathFragment vendorDirectory = options.getOptions(RepositoryOptions.class).vendorDirectory;
110133
LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class);
111134
try {
112-
env.syncPackageLoading(options);
113-
if (!vendorOptions.repos.isEmpty()) {
135+
if (!options.getResidue().isEmpty()) {
136+
result = vendorTargets(env, options, options.getResidue(), vendorDirectory);
137+
} else if (!vendorOptions.repos.isEmpty()) {
114138
result = vendorRepos(env, threadsOption, vendorOptions.repos, vendorDirectory);
115139
} else {
116140
result = vendorAll(env, threadsOption, vendorDirectory);
117141
}
118-
} catch (AbruptExitException e) {
119-
return createFailedBlazeCommandResult(
120-
env.getReporter(), e.getMessage(), e.getDetailedExitCode());
121142
} catch (InterruptedException e) {
122143
return createFailedBlazeCommandResult(
123144
env.getReporter(), "Vendor interrupted: " + e.getMessage());
@@ -165,15 +186,16 @@ private BlazeCommandResult vendorAll(
165186
SkyKey fetchKey = BazelFetchAllValue.key(/* configureEnabled= */ false);
166187
EvaluationResult<SkyValue> evaluationResult =
167188
env.getSkyframeExecutor().prepareAndGet(ImmutableSet.of(fetchKey), evaluationContext);
168-
if (evaluationResult.hasError()) {
169-
Exception e = evaluationResult.getError().getException();
170-
return createFailedBlazeCommandResult(
171-
env.getReporter(),
172-
e != null ? e.getMessage() : "Unexpected error during fetching all external deps.");
173-
}
189+
if (evaluationResult.hasError()) {
190+
Exception e = evaluationResult.getError().getException();
191+
return createFailedBlazeCommandResult(
192+
env.getReporter(),
193+
e != null ? e.getMessage() : "Unexpected error during fetching all external deps.");
194+
}
174195

175196
BazelFetchAllValue fetchAllValue = (BazelFetchAllValue) evaluationResult.get(fetchKey);
176197
vendor(env, vendorDirectory, fetchAllValue.getReposToVendor());
198+
env.getReporter().handle(Event.info("All external dependencies vendored successfully."));
177199
return BlazeCommandResult.success();
178200
}
179201

@@ -212,9 +234,69 @@ private BlazeCommandResult vendorRepos(
212234
return createFailedBlazeCommandResult(
213235
env.getReporter(), "Vendoring some repos failed with errors: " + notFoundRepoErrors);
214236
}
237+
env.getReporter().handle(Event.info("All requested repos vendored successfully."));
215238
return BlazeCommandResult.success();
216239
}
217240

241+
private BlazeCommandResult vendorTargets(
242+
CommandEnvironment env,
243+
OptionsParsingResult options,
244+
List<String> targets,
245+
PathFragment vendorDirectory)
246+
throws InterruptedException, IOException {
247+
// Call fetch which runs build to have the targets graph and configuration set
248+
BuildResult buildResult;
249+
try {
250+
buildResult = TargetFetcher.fetchTargets(env, options, targets);
251+
} catch (TargetFetcherException e) {
252+
return createFailedBlazeCommandResult(
253+
env.getReporter(), Code.QUERY_EVALUATION_ERROR, e.getMessage());
254+
}
255+
256+
// Traverse the graph created from build to collect repos and vendor them
257+
ImmutableList<SkyKey> targetKeys =
258+
buildResult.getActualTargets().stream()
259+
.map(
260+
target ->
261+
ConfiguredTargetKey.builder()
262+
.setConfigurationKey(target.getConfigurationKey())
263+
.setLabel(target.getLabel())
264+
.build())
265+
.collect(toImmutableList());
266+
InMemoryGraph inMemoryGraph = env.getSkyframeExecutor().getEvaluator().getInMemoryGraph();
267+
ImmutableSet<RepositoryName> reposToVendor = collectReposFromTargets(inMemoryGraph, targetKeys);
268+
269+
vendor(env, vendorDirectory, reposToVendor.asList());
270+
env.getReporter()
271+
.handle(
272+
Event.info(
273+
"All External dependencies for the requested targets vendored successfully."));
274+
return BlazeCommandResult.success();
275+
}
276+
277+
private ImmutableSet<RepositoryName> collectReposFromTargets(
278+
InMemoryGraph inMemoryGraph, ImmutableList<SkyKey> targetKeys) throws InterruptedException {
279+
ImmutableSet.Builder<RepositoryName> repos = ImmutableSet.builder();
280+
Queue<SkyKey> nodes = new ArrayDeque<>(targetKeys);
281+
Set<SkyKey> visited = new HashSet<>();
282+
while (!nodes.isEmpty()) {
283+
SkyKey key = nodes.remove();
284+
visited.add(key);
285+
NodeEntry nodeEntry = inMemoryGraph.get(null, Reason.VENDOR_EXTERNAL_REPOS, key);
286+
if (nodeEntry.getValue() instanceof RepositoryDirectoryValue repoDirValue
287+
&& repoDirValue.repositoryExists()
288+
&& !repoDirValue.excludeFromVendoring()) {
289+
repos.add((RepositoryName) key.argument());
290+
}
291+
for (SkyKey depKey : nodeEntry.getDirectDeps()) {
292+
if (!visited.contains(depKey)) {
293+
nodes.add(depKey);
294+
}
295+
}
296+
}
297+
return repos.build();
298+
}
299+
218300
/**
219301
* Copies the fetched repos from the external cache into the vendor directory, unless the repo is
220302
* ignored or was already vendored and up-to-date
@@ -250,6 +332,8 @@ private void vendor(
250332
FileSystemUtils.createEmptyFile(vendorIgnore);
251333
}
252334

335+
env.getReporter().handle(Event.info("Vendoring ..."));
336+
253337
// Update "out-of-date" repos under the vendor directory
254338
for (RepositoryName repo : reposToVendor) {
255339
if (!isRepoUpToDate(repo.getName(), vendorPath, externalPath)) {

‎src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,7 @@ private boolean shouldUseVendorRepos(Environment env, RepositoryFunction handler
392392
return false;
393393
}
394394

395+
// TODO(salmasamy) do we need to check vendor ignore here?
395396
return true;
396397
}
397398

‎src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,9 @@ enum Reason {
221221
/** The node is being looked up to service another "graph lookup" function. */
222222
WALKABLE_GRAPH_OTHER,
223223

224+
/** The node is being looked up to vendor external repos from its dependencies. */
225+
VENDOR_EXTERNAL_REPOS,
226+
224227
/** Some other reason than one of the above that needs the node's value and deps. */
225228
OTHER_NEEDING_VALUE_AND_DEPS,
226229

‎src/test/py/bazel/bzlmod/bazel_vendor_test.py‎

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,18 @@ def testBasicVendoring(self):
102102
self.assertIn('.vendorignore', repos_vendored)
103103

104104
def testVendorFailsWithNofetch(self):
105-
self.ScratchFile('MODULE.bazel')
105+
self.ScratchFile(
106+
'MODULE.bazel',
107+
[
108+
'local_path_override(module_name="bazel_tools", path="tools_mock")',
109+
'local_path_override(module_name="local_config_platform", ',
110+
'path="platforms_mock")',
111+
],
112+
)
106113
self.ScratchFile('BUILD')
114+
# We need to fetch first so that it won't fail while creating the initial
115+
# repo mapping because of --nofetch
116+
self.RunBazel(['fetch', '--all'])
107117
_, _, stderr = self.RunBazel(
108118
['vendor', '--vendor_dir=vendor', '--nofetch'], allow_failure=True
109119
)
@@ -501,6 +511,70 @@ def testBuildingVendoredRepoInOfflineMode(self):
501511
stderr,
502512
)
503513

514+
def testBasicVendorTarget(self):
515+
self.main_registry.createCcModule('aaa', '1.0').createCcModule(
516+
'bbb', '1.0'
517+
).createCcModule('ccc', '1.0')
518+
self.ScratchFile(
519+
'MODULE.bazel',
520+
[
521+
'bazel_dep(name = "aaa", version = "1.0")',
522+
'bazel_dep(name = "bbb", version = "1.0")',
523+
'bazel_dep(name = "ccc", version = "1.0")',
524+
],
525+
)
526+
self.ScratchFile('BUILD')
527+
528+
self.RunBazel(
529+
['vendor', '@aaa//:lib_aaa', '@bbb//:lib_bbb', '--vendor_dir=vendor']
530+
)
531+
# Assert aaa & bbb and are vendored
532+
self.assertIn('aaa~', os.listdir(self._test_cwd + '/vendor'))
533+
self.assertIn('bbb~', os.listdir(self._test_cwd + '/vendor'))
534+
self.assertNotIn('ccc~', os.listdir(self._test_cwd + '/vendor'))
535+
536+
def testVendorTarget(self):
537+
self.main_registry.createCcModule('aaa', '1.0').createCcModule(
538+
'bbb', '1.0', {'aaa': '1.0'}
539+
)
540+
self.ScratchFile(
541+
'MODULE.bazel',
542+
[
543+
'bazel_dep(name = "bbb", version = "1.0")',
544+
],
545+
)
546+
self.ScratchFile(
547+
'BUILD',
548+
[
549+
'cc_binary(',
550+
' name = "main",',
551+
' srcs = ["main.cc"],',
552+
' deps = [',
553+
' "@bbb//:lib_bbb",',
554+
' ],',
555+
')',
556+
],
557+
)
558+
self.ScratchFile(
559+
'main.cc',
560+
[
561+
'#include "aaa.h"',
562+
'int main() {',
563+
' hello_aaa("Hello there!");',
564+
'}',
565+
],
566+
)
567+
568+
self.RunBazel(['vendor', '//:main', '--vendor_dir=vendor'])
569+
570+
# Run the vendored target with --nofetch should only use what is under
571+
# vendor to build, meaning we have vendored everything we need to build/run
572+
# this target
573+
_, stdout, _ = self.RunBazel(
574+
['run', '//:main', '--vendor_dir=vendor', '--nofetch']
575+
)
576+
self.assertIn('Hello there! => aaa@1.0', stdout)
577+
504578

505579
if __name__ == '__main__':
506580
absltest.main()

0 commit comments

Comments
 (0)