Skip to content

Commit 4e29042

Browse files
coeuvrecopybara-github
coeuvre
authored and
copybara-github
committed
Save remote output metadata to ActionCache and load them before checking action cache
This PR extends `ActionCache.Entry` to store output metadata by having a map of <Path, Metadata>. This map is updated after action execution when we update action cache so that metadata of all outputs of the action are saved. Before checking the action cache (when executing actions), we will load the output metadata into output store if it is remote and the correspondingly local one is missing. With this change, remote output metadata is saved to disk so build without bytes can use them among server restarts. We can also download outputs after action execution since remote output metadata can be accessed outside. Part of #12665. Fixes #8248. Closes #13604. PiperOrigin-RevId: 388586691
1 parent f913ef1 commit 4e29042

15 files changed

Lines changed: 1587 additions & 185 deletions

‎src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java‎

Lines changed: 237 additions & 39 deletions
Large diffs are not rendered by default.

‎src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java‎

Lines changed: 153 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,23 @@
1414

1515
package com.google.devtools.build.lib.actions.cache;
1616

17-
import com.google.common.base.Preconditions;
17+
import static com.google.common.base.Preconditions.checkArgument;
18+
import static com.google.common.base.Preconditions.checkState;
19+
import static com.google.common.collect.ImmutableMap.toImmutableMap;
20+
21+
import com.google.auto.value.AutoValue;
1822
import com.google.common.collect.ImmutableList;
1923
import com.google.common.collect.ImmutableMap;
2024
import com.google.common.collect.Lists;
2125
import com.google.common.io.BaseEncoding;
26+
import com.google.devtools.build.lib.actions.Artifact;
27+
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
2228
import com.google.devtools.build.lib.actions.FileArtifactValue;
29+
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
2330
import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics;
2431
import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason;
2532
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
33+
import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
2634
import com.google.devtools.build.lib.vfs.PathFragment;
2735
import java.io.IOException;
2836
import java.io.PrintStream;
@@ -32,6 +40,7 @@
3240
import java.util.HashMap;
3341
import java.util.List;
3442
import java.util.Map;
43+
import java.util.Optional;
3544
import javax.annotation.Nullable;
3645

3746
/**
@@ -84,31 +93,148 @@ final class Entry {
8493
private Map<String, FileArtifactValue> mdMap;
8594
private byte[] digest;
8695
private final byte[] usedClientEnvDigest;
96+
private final Map<String, RemoteFileArtifactValue> outputFileMetadata;
97+
private final Map<String, SerializableTreeArtifactValue> outputTreeMetadata;
98+
99+
/**
100+
* The metadata for output tree that can be serialized.
101+
*
102+
* <p>We can't serialize {@link TreeArtifactValue} directly as it contains some objects that we
103+
* don't want to serialize, e.g. {@link SpecialArtifact}.
104+
*/
105+
@AutoValue
106+
public abstract static class SerializableTreeArtifactValue {
107+
public static SerializableTreeArtifactValue create(
108+
ImmutableMap<String, RemoteFileArtifactValue> childValues,
109+
Optional<RemoteFileArtifactValue> archivedFileValue) {
110+
return new AutoValue_ActionCache_Entry_SerializableTreeArtifactValue(
111+
childValues, archivedFileValue);
112+
}
113+
114+
/**
115+
* Creates {@link SerializableTreeArtifactValue} from {@link TreeArtifactValue} by collecting
116+
* children and archived artifact which are remote.
117+
*
118+
* <p>If no remote value, {@link Optional#empty} is returned.
119+
*/
120+
public static Optional<SerializableTreeArtifactValue> createSerializable(
121+
TreeArtifactValue treeMetadata) {
122+
ImmutableMap<String, RemoteFileArtifactValue> childValues =
123+
treeMetadata.getChildValues().entrySet().stream()
124+
// Only save remote tree file
125+
.filter(e -> e.getValue().isRemote())
126+
.collect(
127+
toImmutableMap(
128+
e -> e.getKey().getTreeRelativePathString(),
129+
e -> (RemoteFileArtifactValue) e.getValue()));
130+
131+
// Only save remote archived artifact
132+
Optional<RemoteFileArtifactValue> archivedFileValue =
133+
treeMetadata
134+
.getArchivedRepresentation()
135+
.filter(ar -> ar.archivedFileValue().isRemote())
136+
.map(ar -> (RemoteFileArtifactValue) ar.archivedFileValue());
137+
138+
if (childValues.isEmpty() && !archivedFileValue.isPresent()) {
139+
return Optional.empty();
140+
}
141+
142+
return Optional.of(SerializableTreeArtifactValue.create(childValues, archivedFileValue));
143+
}
144+
145+
// A map from parentRelativePath to the file metadata
146+
public abstract ImmutableMap<String, RemoteFileArtifactValue> childValues();
147+
148+
public abstract Optional<RemoteFileArtifactValue> archivedFileValue();
149+
}
87150

88151
public Entry(String key, Map<String, String> usedClientEnv, boolean discoversInputs) {
89152
actionKey = key;
90153
this.usedClientEnvDigest = MetadataDigestUtils.fromEnv(usedClientEnv);
91154
files = discoversInputs ? new ArrayList<String>() : null;
92155
mdMap = new HashMap<>();
156+
outputFileMetadata = new HashMap<>();
157+
outputTreeMetadata = new HashMap<>();
93158
}
94159

95160
public Entry(
96-
String key, byte[] usedClientEnvDigest, @Nullable List<String> files, byte[] digest) {
161+
String key,
162+
byte[] usedClientEnvDigest,
163+
@Nullable List<String> files,
164+
byte[] digest,
165+
Map<String, RemoteFileArtifactValue> outputFileMetadata,
166+
Map<String, SerializableTreeArtifactValue> outputTreeMetadata) {
97167
actionKey = key;
98168
this.usedClientEnvDigest = usedClientEnvDigest;
99169
this.files = files;
100170
this.digest = digest;
101171
mdMap = null;
172+
this.outputFileMetadata = outputFileMetadata;
173+
this.outputTreeMetadata = outputTreeMetadata;
102174
}
103175

104-
/**
105-
* Adds the artifact, specified by the executable relative path and its metadata into the cache
106-
* entry.
107-
*/
108-
public void addFile(PathFragment relativePath, FileArtifactValue md, boolean saveExecPath) {
109-
Preconditions.checkState(mdMap != null);
110-
Preconditions.checkState(!isCorrupted());
111-
Preconditions.checkState(digest == null);
176+
/** Adds metadata of an output file */
177+
public void addOutputFile(Artifact output, FileArtifactValue value, boolean saveFileMetadata) {
178+
checkArgument(
179+
!output.isTreeArtifact() && !output.isChildOfDeclaredDirectory(),
180+
"Must use addOutputTree to save tree artifacts and their children: %s",
181+
output);
182+
checkState(mdMap != null);
183+
checkState(!isCorrupted());
184+
checkState(digest == null);
185+
186+
String execPath = output.getExecPathString();
187+
// Only save remote file metadata
188+
if (saveFileMetadata && value.isRemote()) {
189+
outputFileMetadata.put(execPath, (RemoteFileArtifactValue) value);
190+
}
191+
mdMap.put(execPath, value);
192+
}
193+
194+
/** Gets metadata of an output file */
195+
@Nullable
196+
public RemoteFileArtifactValue getOutputFile(Artifact output) {
197+
checkState(!isCorrupted());
198+
return outputFileMetadata.get(output.getExecPathString());
199+
}
200+
201+
Map<String, RemoteFileArtifactValue> getOutputFiles() {
202+
return outputFileMetadata;
203+
}
204+
205+
/** Adds metadata of an output tree */
206+
public void addOutputTree(
207+
SpecialArtifact output, TreeArtifactValue metadata, boolean saveTreeMetadata) {
208+
checkArgument(output.isTreeArtifact(), "artifact must be a tree artifact: %s", output);
209+
checkState(mdMap != null);
210+
checkState(!isCorrupted());
211+
checkState(digest == null);
212+
213+
String execPath = output.getExecPathString();
214+
if (saveTreeMetadata) {
215+
SerializableTreeArtifactValue.createSerializable(metadata)
216+
.ifPresent(value -> outputTreeMetadata.put(execPath, value));
217+
}
218+
mdMap.put(execPath, metadata.getMetadata());
219+
}
220+
221+
/** Gets metadata of an output tree */
222+
@Nullable
223+
public SerializableTreeArtifactValue getOutputTree(SpecialArtifact output) {
224+
checkState(!isCorrupted());
225+
return outputTreeMetadata.get(output.getExecPathString());
226+
}
227+
228+
Map<String, SerializableTreeArtifactValue> getOutputTrees() {
229+
return outputTreeMetadata;
230+
}
231+
232+
/** Adds metadata of an input file */
233+
public void addInputFile(
234+
PathFragment relativePath, FileArtifactValue md, boolean saveExecPath) {
235+
checkState(mdMap != null);
236+
checkState(!isCorrupted());
237+
checkState(digest == null);
112238

113239
String execPath = relativePath.getPathString();
114240
if (discoversInputs() && saveExecPath) {
@@ -117,8 +243,8 @@ public void addFile(PathFragment relativePath, FileArtifactValue md, boolean sav
117243
mdMap.put(execPath, md);
118244
}
119245

120-
public void addFile(PathFragment relativePath, FileArtifactValue md) {
121-
addFile(relativePath, md, /* saveExecPath= */ true);
246+
public void addInputFile(PathFragment relativePath, FileArtifactValue md) {
247+
addInputFile(relativePath, md, /*saveExecPath=*/ true);
122248
}
123249

124250
/**
@@ -197,6 +323,21 @@ public String toString() {
197323
builder.append(" ").append(info).append("\n");
198324
}
199325
}
326+
327+
for (Map.Entry<String, RemoteFileArtifactValue> entry : outputFileMetadata.entrySet()) {
328+
builder
329+
.append(" ")
330+
.append(entry.getKey())
331+
.append(" = ")
332+
.append(entry.getValue())
333+
.append("\n");
334+
}
335+
336+
for (Map.Entry<String, SerializableTreeArtifactValue> entry : outputTreeMetadata.entrySet()) {
337+
SerializableTreeArtifactValue metadata = entry.getValue();
338+
builder.append(" ").append(entry.getKey()).append(" = ").append(metadata).append("\n");
339+
}
340+
200341
return builder.toString();
201342
}
202343
}

0 commit comments

Comments
 (0)