Skip to content

Commit e7635c5

Browse files
SalmaSamymeteorcloudy
authored and
copybara-github
committed
VENDOR.bazel file (related #19563)
A Starlark config file to allow controlling the vendored repos via: - ignore('repo_name','repo_name', ....) function --> Bazel will never vendor the repo or consider this directory while building in vendor mode. - pin('repo_name','repo_name', ....) function --> Bazel will pin the contents of this repo, will not update it while vendoring, and will always use it as is when building in vendor mode Co-authored-by: Yun Peng <pcloudy@google.com> PiperOrigin-RevId: 639000162 Change-Id: I278359f6e92eab290dac9cc4ef3136aa140cf8ba
1 parent 2fb8564 commit e7635c5

14 files changed

Lines changed: 539 additions & 65 deletions

File tree

‎src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkGlobalsImpl.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import com.google.devtools.build.lib.packages.StarlarkGlobals;
2929
import com.google.devtools.build.lib.packages.StarlarkNativeModule;
3030
import com.google.devtools.build.lib.packages.StructProvider;
31+
import com.google.devtools.build.lib.packages.VendorFileGlobals;
3132
import net.starlark.java.eval.Starlark;
3233
import net.starlark.java.lib.json.Json;
3334

@@ -135,4 +136,11 @@ public ImmutableMap<String, Object> getRepoToplevels() {
135136
Starlark.addMethods(env, RepoCallable.INSTANCE);
136137
return env.buildOrThrow();
137138
}
139+
140+
@Override
141+
public ImmutableMap<String, Object> getVendorToplevels() {
142+
ImmutableMap.Builder<String, Object> env = ImmutableMap.builder();
143+
Starlark.addMethods(env, VendorFileGlobals.INSTANCE);
144+
return env.buildOrThrow();
145+
}
138146
}

‎src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalFunction;
5555
import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionFunction;
5656
import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionUsagesFunction;
57+
import com.google.devtools.build.lib.bazel.bzlmod.VendorFileFunction;
5758
import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsFunction;
5859
import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil;
5960
import com.google.devtools.build.lib.bazel.commands.FetchCommand;
@@ -278,6 +279,9 @@ SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace
278279
directories.getWorkspace()))
279280
.addSkyFunction(SkyFunctions.REPO_SPEC, new RepoSpecFunction())
280281
.addSkyFunction(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction())
282+
.addSkyFunction(
283+
SkyFunctions.VENDOR_FILE,
284+
new VendorFileFunction(runtime.getRuleClassProvider().getBazelStarlarkEnvironment()))
281285
.addSkyFunction(
282286
SkyFunctions.MODULE_EXTENSION_REPO_MAPPING_ENTRIES,
283287
new ModuleExtensionRepoMappingEntriesFunction());
@@ -602,6 +606,7 @@ public ImmutableList<Injected> getPrecomputedValues() {
602606
PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode),
603607
PrecomputedValue.injected(RepositoryDelegatorFunction.IS_VENDOR_COMMAND, false),
604608
PrecomputedValue.injected(RepositoryDelegatorFunction.VENDOR_DIRECTORY, vendorDirectory),
609+
PrecomputedValue.injected(VendorFileFunction.VENDOR_DIRECTORY, vendorDirectory),
605610
PrecomputedValue.injected(
606611
YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, allowedYankedVersions),
607612
PrecomputedValue.injected(

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ java_library(
146146
"SingleExtensionUsagesValue.java",
147147
"SingleExtensionValue.java",
148148
"SingleVersionOverride.java",
149+
"VendorFileValue.java",
149150
"YankedVersionsValue.java",
150151
],
151152
deps = [
@@ -205,6 +206,7 @@ java_library(
205206
"SingleExtensionUsagesFunction.java",
206207
"StarlarkBazelModule.java",
207208
"TypeCheckedTag.java",
209+
"VendorFileFunction.java",
208210
"YankedVersionsFunction.java",
209211
"YankedVersionsUtil.java",
210212
],

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileFunction.java‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
// Copyright 2024 The Bazel Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.google.devtools.build.lib.bazel.bzlmod;
16+
17+
import static java.nio.charset.StandardCharsets.UTF_8;
18+
19+
import com.google.common.collect.ImmutableList;
20+
import com.google.devtools.build.lib.actions.FileValue;
21+
import com.google.devtools.build.lib.cmdline.LabelConstants;
22+
import com.google.devtools.build.lib.events.Event;
23+
import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment;
24+
import com.google.devtools.build.lib.packages.DotBazelFileSyntaxChecker;
25+
import com.google.devtools.build.lib.packages.VendorThreadContext;
26+
import com.google.devtools.build.lib.skyframe.PrecomputedValue;
27+
import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed;
28+
import com.google.devtools.build.lib.vfs.FileSystemUtils;
29+
import com.google.devtools.build.lib.vfs.Path;
30+
import com.google.devtools.build.lib.vfs.Root;
31+
import com.google.devtools.build.lib.vfs.RootedPath;
32+
import com.google.devtools.build.skyframe.SkyFunction;
33+
import com.google.devtools.build.skyframe.SkyFunctionException;
34+
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
35+
import com.google.devtools.build.skyframe.SkyKey;
36+
import com.google.devtools.build.skyframe.SkyValue;
37+
import java.io.IOException;
38+
import java.util.Optional;
39+
import javax.annotation.Nullable;
40+
import net.starlark.java.eval.EvalException;
41+
import net.starlark.java.eval.Mutability;
42+
import net.starlark.java.eval.Starlark;
43+
import net.starlark.java.eval.StarlarkSemantics;
44+
import net.starlark.java.eval.StarlarkThread;
45+
import net.starlark.java.eval.SymbolGenerator;
46+
import net.starlark.java.syntax.ParserInput;
47+
import net.starlark.java.syntax.Program;
48+
import net.starlark.java.syntax.StarlarkFile;
49+
import net.starlark.java.syntax.SyntaxError;
50+
51+
/**
52+
* The function to evaluate the VENDOR.bazel file under the vendor directory specified by the flag:
53+
* --vendor_dir.
54+
*/
55+
public class VendorFileFunction implements SkyFunction {
56+
57+
public static final Precomputed<Optional<Path>> VENDOR_DIRECTORY =
58+
new Precomputed<>("vendor_directory");
59+
60+
private static final String VENDOR_FILE_HEADER =
61+
"""
62+
###############################################################################
63+
# This file is used to configure how external repositories are handled in vendor mode.
64+
# ONLY the two following functions can be used:
65+
#
66+
# ignore('@@<canonical repo name>', ...) is used to completely ignore this repo from vendoring.
67+
# Bazel will use the normal external cache and fetch process for this repo.
68+
#
69+
# pin('@@<canonical repo name>', ...) is used to pin the contents of this repo under the vendor
70+
# directory as if there is a --override_repository flag for this repo.
71+
# Note that Bazel will NOT update the vendored source for this repo while running vendor command
72+
# unless it's unpinned. The user can modify and maintain the vendored source for this repo manually.
73+
###############################################################################
74+
""";
75+
76+
private final BazelStarlarkEnvironment starlarkEnv;
77+
78+
public VendorFileFunction(BazelStarlarkEnvironment starlarkEnv) {
79+
this.starlarkEnv = starlarkEnv;
80+
}
81+
82+
@Nullable
83+
@Override
84+
public SkyValue compute(SkyKey skyKey, Environment env)
85+
throws SkyFunctionException, InterruptedException {
86+
if (VENDOR_DIRECTORY.get(env).isEmpty()) {
87+
throw new VendorFileFunctionException(
88+
new IllegalStateException(
89+
"VENDOR.bazel file is not accessible with vendor mode off (without --vendor_dir"
90+
+ " flag)"),
91+
Transience.PERSISTENT);
92+
}
93+
94+
Path vendorPath = VENDOR_DIRECTORY.get(env).get();
95+
RootedPath vendorFilePath =
96+
RootedPath.toRootedPath(Root.fromPath(vendorPath), LabelConstants.VENDOR_FILE_NAME);
97+
98+
FileValue vendorFileValue = (FileValue) env.getValue(FileValue.key(vendorFilePath));
99+
if (vendorFileValue == null) {
100+
return null;
101+
}
102+
if (!vendorFileValue.exists()) {
103+
createVendorFile(vendorPath, vendorFilePath.asPath());
104+
return VendorFileValue.create(ImmutableList.of(), ImmutableList.of());
105+
}
106+
107+
StarlarkSemantics starlarkSemantics = PrecomputedValue.STARLARK_SEMANTICS.get(env);
108+
if (starlarkSemantics == null) {
109+
return null;
110+
}
111+
VendorThreadContext context =
112+
getVendorFileContext(env, skyKey, vendorFilePath.asPath(), starlarkSemantics);
113+
return VendorFileValue.create(context.getIgnoredRepos(), context.getPinnedRepos());
114+
}
115+
116+
private VendorThreadContext getVendorFileContext(
117+
Environment env, SkyKey skyKey, Path vendorFilePath, StarlarkSemantics starlarkSemantics)
118+
throws VendorFileFunctionException, InterruptedException {
119+
try (Mutability mu = Mutability.create("vendor file")) {
120+
StarlarkFile vendorFile = readAndParseVendorFile(vendorFilePath, env);
121+
new DotBazelFileSyntaxChecker("VENDOR.bazel files", /* canLoadBzl= */ false)
122+
.check(vendorFile);
123+
net.starlark.java.eval.Module predeclaredEnv =
124+
net.starlark.java.eval.Module.withPredeclared(
125+
starlarkSemantics, starlarkEnv.getStarlarkGlobals().getVendorToplevels());
126+
Program program = Program.compileFile(vendorFile, predeclaredEnv);
127+
StarlarkThread thread =
128+
StarlarkThread.create(
129+
mu, starlarkSemantics, /* contextDescription= */ "", SymbolGenerator.create(skyKey));
130+
VendorThreadContext context = new VendorThreadContext();
131+
context.storeInThread(thread);
132+
Starlark.execFileProgram(program, predeclaredEnv, thread);
133+
return context;
134+
} catch (SyntaxError.Exception | EvalException e) {
135+
throw new VendorFileFunctionException(
136+
new BadVendorFileException("error parsing VENDOR.bazel file: " + e.getMessage()),
137+
Transience.PERSISTENT);
138+
}
139+
}
140+
141+
private void createVendorFile(Path vendorPath, Path vendorFilePath)
142+
throws VendorFileFunctionException {
143+
try {
144+
vendorPath.createDirectoryAndParents();
145+
byte[] vendorFileContents = VENDOR_FILE_HEADER.getBytes(UTF_8);
146+
FileSystemUtils.writeContent(vendorFilePath, vendorFileContents);
147+
} catch (IOException e) {
148+
throw new VendorFileFunctionException(
149+
new IOException("error creating VENDOR.bazel file", e), Transience.TRANSIENT);
150+
}
151+
}
152+
153+
private static StarlarkFile readAndParseVendorFile(Path path, Environment env)
154+
throws VendorFileFunctionException {
155+
byte[] contents;
156+
try {
157+
contents = FileSystemUtils.readWithKnownFileSize(path, path.getFileSize());
158+
} catch (IOException e) {
159+
throw new VendorFileFunctionException(
160+
new IOException("error reading VENDOR.bazel file", e), Transience.TRANSIENT);
161+
}
162+
StarlarkFile starlarkFile =
163+
StarlarkFile.parse(ParserInput.fromUTF8(contents, path.getPathString()));
164+
if (!starlarkFile.ok()) {
165+
Event.replayEventsOn(env.getListener(), starlarkFile.errors());
166+
throw new VendorFileFunctionException(
167+
new BadVendorFileException("error parsing VENDOR.bazel file"), Transience.PERSISTENT);
168+
}
169+
return starlarkFile;
170+
}
171+
172+
/** Thrown when something is wrong with the contents of the VENDOR.bazel file. */
173+
public static class BadVendorFileException extends Exception {
174+
public BadVendorFileException(String message) {
175+
super(message);
176+
}
177+
}
178+
179+
static class VendorFileFunctionException extends SkyFunctionException {
180+
private VendorFileFunctionException(Exception e, Transience transience) {
181+
super(e, transience);
182+
}
183+
}
184+
}

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileValue.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright 2024 The Bazel Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.google.devtools.build.lib.bazel.bzlmod;
16+
17+
import com.google.auto.value.AutoValue;
18+
import com.google.common.collect.ImmutableList;
19+
import com.google.devtools.build.lib.cmdline.RepositoryName;
20+
import com.google.devtools.build.lib.skyframe.SkyFunctions;
21+
import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant;
22+
import com.google.devtools.build.skyframe.SkyKey;
23+
import com.google.devtools.build.skyframe.SkyValue;
24+
25+
/** Represent the parsed VENDOR.bazel file */
26+
@AutoValue
27+
public abstract class VendorFileValue implements SkyValue {
28+
29+
@SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.VENDOR_FILE;
30+
31+
public abstract ImmutableList<RepositoryName> getIgnoredRepos();
32+
33+
public abstract ImmutableList<RepositoryName> getPinnedRepos();
34+
35+
public static VendorFileValue create(
36+
ImmutableList<RepositoryName> ignoredRepos, ImmutableList<RepositoryName> pinnedRepos) {
37+
return new AutoValue_VendorFileValue(ignoredRepos, pinnedRepos);
38+
}
39+
}

‎src/main/java/com/google/devtools/build/lib/bazel/commands/VendorCommand.java‎

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,16 @@
7373
import java.util.Set;
7474
import javax.annotation.Nullable;
7575

76-
/** Fetches external repositories into a specified directory. */
76+
/**
77+
* Fetches external repositories into a specified directory.
78+
*
79+
* <p>This command is used to fetch external repositories into a specified directory. It can be used
80+
* to fetch all external repositories, a specific list of repositories or the repositories needed to
81+
* build a specific list of targets.
82+
*
83+
* <p>The command is used to create a vendor directory that can be used to build the project
84+
* offline.
85+
*/
7786
@Command(
7887
name = VendorCommand.NAME,
7988
builds = true,
@@ -93,9 +102,6 @@
93102
public final class VendorCommand implements BlazeCommand {
94103
public static final String NAME = "vendor";
95104

96-
// TODO(salmasamy) decide on name and format
97-
private static final String VENDOR_IGNORE = ".vendorignore";
98-
99105
@Override
100106
public void editOptions(OptionsParser optionsParser) {
101107
// We only need to inject these options with fetch target (when there is a residue)
@@ -270,7 +276,7 @@ private BlazeCommandResult vendorTargets(
270276
env.getReporter()
271277
.handle(
272278
Event.info(
273-
"All External dependencies for the requested targets vendored successfully."));
279+
"All external dependencies for the requested targets vendored successfully."));
274280
return BlazeCommandResult.success();
275281
}
276282

@@ -314,24 +320,11 @@ private void vendor(
314320
env.getDirectories()
315321
.getOutputBase()
316322
.getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION);
317-
Path vendorIgnore = vendorPath.getRelative(VENDOR_IGNORE);
318323

319324
if (!vendorPath.exists()) {
320325
vendorPath.createDirectory();
321326
}
322327

323-
// exclude any ignored repo under .vendorignore
324-
if (vendorIgnore.exists()) {
325-
ImmutableSet<String> ignoredRepos =
326-
ImmutableSet.copyOf(FileSystemUtils.readLines(vendorIgnore, UTF_8));
327-
reposToVendor =
328-
reposToVendor.stream()
329-
.filter(repo -> !ignoredRepos.contains(repo.getName()))
330-
.collect(toImmutableList());
331-
} else {
332-
FileSystemUtils.createEmptyFile(vendorIgnore);
333-
}
334-
335328
env.getReporter().handle(Event.info("Vendoring ..."));
336329

337330
// Update "out-of-date" repos under the vendor directory
@@ -361,7 +354,7 @@ private boolean isRepoUpToDate(String repoName, Path vendorPath, Path externalPa
361354
return false;
362355
}
363356

364-
// Since this runs after BazelFetchAllFunction, its guaranteed that the marker files
357+
// Since this runs after fetching repos, its guaranteed that the marker files
365358
// under $OUTPUT_BASE/external are up-to-date. We just need to compare it against the marker
366359
// under vendor.
367360
Path externalMarkerFile = externalPath.getChild("@" + repoName + ".marker");

‎src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public class LabelConstants {
4545
PathFragment.create("WORKSPACE.bazel");
4646
public static final PathFragment MODULE_DOT_BAZEL_FILE_NAME = PathFragment.create("MODULE.bazel");
4747
public static final PathFragment REPO_FILE_NAME = PathFragment.create("REPO.bazel");
48+
public static final PathFragment VENDOR_FILE_NAME = PathFragment.create("VENDOR.bazel");
4849

4950
public static final PathFragment MODULE_LOCKFILE_NAME = PathFragment.create("MODULE.bazel.lock");
5051

‎src/main/java/com/google/devtools/build/lib/packages/StarlarkGlobals.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,4 +70,7 @@ public interface StarlarkGlobals {
7070

7171
/** Returns the top-levels for REPO.bazel files. */
7272
ImmutableMap<String, Object> getRepoToplevels();
73+
74+
/** Returns the top-levels for VENDOR.bazel files. */
75+
ImmutableMap<String, Object> getVendorToplevels();
7376
}

0 commit comments

Comments
 (0)