Skip to content

Commit 799e03e

Browse files
lhotarilhotari
committed
intern duplicate file path Strings stored in the in-memory caches
- uses Guava's Interner Implementation +review REVIEW-5656
1 parent d542d59 commit 799e03e

17 files changed

Lines changed: 178 additions & 40 deletions

‎subprojects/core/src/main/groovy/org/gradle/api/internal/cache/StringInterner.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Copyright 2015 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.gradle.api.internal.cache;
18+
19+
import com.google.common.collect.Interner;
20+
import com.google.common.collect.Interners;
21+
22+
public class StringInterner implements Interner<String> {
23+
private final Interner<String> interner;
24+
25+
public StringInterner() {
26+
this.interner = Interners.newWeakInterner();
27+
}
28+
29+
@Override
30+
public String intern(String sample) {
31+
if (sample == null) {
32+
return null;
33+
}
34+
return interner.intern(sample);
35+
}
36+
}

‎subprojects/core/src/main/groovy/org/gradle/api/internal/changedetection/state/CacheBackedTaskHistoryRepository.java‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package org.gradle.api.internal.changedetection.state;
1717

1818
import org.gradle.api.internal.TaskInternal;
19+
import org.gradle.api.internal.cache.StringInterner;
1920
import org.gradle.cache.PersistentIndexedCache;
2021
import org.gradle.internal.Factory;
2122
import org.gradle.internal.serialize.Decoder;
@@ -29,11 +30,14 @@ public class CacheBackedTaskHistoryRepository implements TaskHistoryRepository {
2930
private final TaskArtifactStateCacheAccess cacheAccess;
3031
private final FileSnapshotRepository snapshotRepository;
3132
private final PersistentIndexedCache<String, TaskHistory> taskHistoryCache;
32-
private final TaskHistorySerializer serializer = new TaskHistorySerializer();
33+
private final TaskHistorySerializer serializer;
34+
private final StringInterner stringInterner;
3335

34-
public CacheBackedTaskHistoryRepository(TaskArtifactStateCacheAccess cacheAccess, FileSnapshotRepository snapshotRepository) {
36+
public CacheBackedTaskHistoryRepository(TaskArtifactStateCacheAccess cacheAccess, FileSnapshotRepository snapshotRepository, StringInterner stringInterner) {
3537
this.cacheAccess = cacheAccess;
3638
this.snapshotRepository = snapshotRepository;
39+
this.stringInterner = stringInterner;
40+
this.serializer = new TaskHistorySerializer(stringInterner);
3741
taskHistoryCache = cacheAccess.createCache("taskArtifacts", String.class, serializer);
3842
}
3943

@@ -100,10 +104,10 @@ public TaskHistory create() {
100104
});
101105
}
102106

103-
private static Set<String> outputFiles(TaskInternal task) {
107+
private Set<String> outputFiles(TaskInternal task) {
104108
Set<String> outputFiles = new HashSet<String>();
105109
for (File file : task.getOutputs().getFiles()) {
106-
outputFiles.add(file.getAbsolutePath());
110+
outputFiles.add(stringInterner.intern(file.getAbsolutePath()));
107111
}
108112
return outputFiles;
109113
}
@@ -136,11 +140,16 @@ private LazyTaskExecution findPreviousExecution(TaskExecution currentExecution,
136140
private static class TaskHistorySerializer implements Serializer<TaskHistory> {
137141

138142
private ClassLoader classLoader;
143+
private final StringInterner stringInterner;
144+
145+
public TaskHistorySerializer(StringInterner stringInterner) {
146+
this.stringInterner = stringInterner;
147+
}
139148

140149
public TaskHistory read(Decoder decoder) throws Exception {
141150
byte executions = decoder.readByte();
142151
TaskHistory history = new TaskHistory();
143-
LazyTaskExecution.TaskHistorySerializer executionSerializer = new LazyTaskExecution.TaskHistorySerializer(classLoader);
152+
LazyTaskExecution.TaskHistorySerializer executionSerializer = new LazyTaskExecution.TaskHistorySerializer(classLoader, stringInterner);
144153
for (int i = 0; i < executions; i++) {
145154
LazyTaskExecution exec = executionSerializer.read(decoder);
146155
history.configurations.add(exec);
@@ -151,7 +160,7 @@ public TaskHistory read(Decoder decoder) throws Exception {
151160
public void write(Encoder encoder, TaskHistory value) throws Exception {
152161
int size = value.configurations.size();
153162
encoder.writeByte((byte) size);
154-
LazyTaskExecution.TaskHistorySerializer executionSerializer = new LazyTaskExecution.TaskHistorySerializer(classLoader);
163+
LazyTaskExecution.TaskHistorySerializer executionSerializer = new LazyTaskExecution.TaskHistorySerializer(classLoader, stringInterner);
155164
for (LazyTaskExecution execution : value.configurations) {
156165
executionSerializer.write(encoder, execution);
157166
}
@@ -228,10 +237,12 @@ public void setOutputFilesSnapshot(FileCollectionSnapshot outputFilesSnapshot) {
228237
}
229238

230239
static class TaskHistorySerializer implements Serializer<LazyTaskExecution> {
231-
private InputPropertiesSerializer inputPropertiesSerializer;
240+
private final InputPropertiesSerializer inputPropertiesSerializer;
241+
private final StringInterner stringInterner;
232242

233-
public TaskHistorySerializer(ClassLoader classLoader) {
243+
public TaskHistorySerializer(ClassLoader classLoader, StringInterner stringInterner) {
234244
this.inputPropertiesSerializer = new InputPropertiesSerializer(classLoader);
245+
this.stringInterner = stringInterner;
235246
}
236247

237248
public LazyTaskExecution read(Decoder decoder) throws Exception {
@@ -242,7 +253,7 @@ public LazyTaskExecution read(Decoder decoder) throws Exception {
242253
int outputFiles = decoder.readInt();
243254
Set<String> files = new HashSet<String>();
244255
for (int j = 0; j < outputFiles; j++) {
245-
files.add(decoder.readString());
256+
files.add(stringInterner.intern(decoder.readString()));
246257
}
247258
execution.setOutputFiles(files);
248259

‎subprojects/core/src/main/groovy/org/gradle/api/internal/changedetection/state/CachingFileSnapshotter.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package org.gradle.api.internal.changedetection.state;
1717

1818
import org.gradle.api.file.FileTreeElement;
19+
import org.gradle.api.internal.cache.StringInterner;
1920
import org.gradle.api.internal.hash.Hasher;
2021
import org.gradle.cache.PersistentIndexedCache;
2122
import org.gradle.cache.PersistentStore;
@@ -29,10 +30,12 @@ public class CachingFileSnapshotter implements FileSnapshotter, FileTreeElementS
2930
private final PersistentIndexedCache<String, FileInfo> cache;
3031
private final Hasher hasher;
3132
private final FileInfoSerializer serializer = new FileInfoSerializer();
33+
private final StringInterner stringInterner;
3234

33-
public CachingFileSnapshotter(Hasher hasher, PersistentStore store) {
35+
public CachingFileSnapshotter(Hasher hasher, PersistentStore store, StringInterner stringInterner) {
3436
this.hasher = hasher;
3537
this.cache = store.createCache("fileHashes", String.class, serializer);
38+
this.stringInterner = stringInterner;
3639
}
3740

3841
public FileInfo snapshot(File file) {
@@ -56,7 +59,7 @@ private FileInfo snapshot(FileWithMetadata fileWithMetadata) {
5659

5760
byte[] hash = hasher.hash(file);
5861
info = new FileInfo(hash, length, timestamp);
59-
cache.put(absolutePath, info);
62+
cache.put(stringInterner.intern(absolutePath), info);
6063
return info;
6164
}
6265

‎subprojects/core/src/main/groovy/org/gradle/api/internal/changedetection/state/DefaultFileCollectionSnapshotter.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.gradle.api.file.FileCollection;
2020
import org.gradle.api.file.FileVisitDetails;
2121
import org.gradle.api.file.FileVisitor;
22+
import org.gradle.api.internal.cache.StringInterner;
2223
import org.gradle.api.internal.file.CachingFileVisitDetails;
2324
import org.gradle.api.internal.file.FileTreeInternal;
2425
import org.gradle.api.internal.file.collections.*;
@@ -33,14 +34,16 @@
3334
public class DefaultFileCollectionSnapshotter implements FileCollectionSnapshotter {
3435
private final FileTreeElementSnapshotter snapshotter;
3536
private TaskArtifactStateCacheAccess cacheAccess;
37+
private final StringInterner stringInterner;
3638

37-
public DefaultFileCollectionSnapshotter(FileTreeElementSnapshotter snapshotter, TaskArtifactStateCacheAccess cacheAccess) {
39+
public DefaultFileCollectionSnapshotter(FileTreeElementSnapshotter snapshotter, TaskArtifactStateCacheAccess cacheAccess, StringInterner stringInterner) {
3840
this.snapshotter = snapshotter;
3941
this.cacheAccess = cacheAccess;
42+
this.stringInterner = stringInterner;
4043
}
4144

4245
public void registerSerializers(SerializerRegistry<FileCollectionSnapshot> registry) {
43-
registry.register(FileCollectionSnapshotImpl.class, new DefaultFileSnapshotterSerializer());
46+
registry.register(FileCollectionSnapshotImpl.class, new DefaultFileSnapshotterSerializer(stringInterner));
4447
}
4548

4649
public FileCollectionSnapshot emptySnapshot() {
@@ -59,7 +62,7 @@ public FileCollectionSnapshot snapshot(final FileCollection input) {
5962
cacheAccess.useCache("Create file snapshot", new Runnable() {
6063
public void run() {
6164
for (FileVisitDetails fileDetails : allFileVisitDetails) {
62-
final String absolutePath = fileDetails.getFile().getAbsolutePath();
65+
final String absolutePath = stringInterner.intern(fileDetails.getFile().getAbsolutePath());
6366
if (!snapshots.containsKey(absolutePath)) {
6467
if (fileDetails.isDirectory()) {
6568
snapshots.put(absolutePath, new DirSnapshot());

‎subprojects/core/src/main/groovy/org/gradle/api/internal/changedetection/state/DefaultFileSnapshotterSerializer.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package org.gradle.api.internal.changedetection.state;
1818

19+
import org.gradle.api.internal.cache.StringInterner;
1920
import org.gradle.internal.serialize.Decoder;
2021
import org.gradle.internal.serialize.Encoder;
2122
import org.gradle.internal.serialize.Serializer;
@@ -24,12 +25,18 @@
2425
import java.util.Map;
2526

2627
class DefaultFileSnapshotterSerializer implements Serializer<DefaultFileCollectionSnapshotter.FileCollectionSnapshotImpl> {
28+
private final StringInterner stringInterner;
29+
30+
public DefaultFileSnapshotterSerializer(StringInterner stringInterner) {
31+
this.stringInterner = stringInterner;
32+
}
33+
2734
public DefaultFileCollectionSnapshotter.FileCollectionSnapshotImpl read(Decoder decoder) throws Exception {
2835
Map<String, DefaultFileCollectionSnapshotter.IncrementalFileSnapshot> snapshots = new HashMap<String, DefaultFileCollectionSnapshotter.IncrementalFileSnapshot>();
2936
DefaultFileCollectionSnapshotter.FileCollectionSnapshotImpl snapshot = new DefaultFileCollectionSnapshotter.FileCollectionSnapshotImpl(snapshots);
3037
int snapshotsCount = decoder.readSmallInt();
3138
for (int i = 0; i < snapshotsCount; i++) {
32-
String key = decoder.readString();
39+
String key = stringInterner.intern(decoder.readString());
3340
byte fileSnapshotKind = decoder.readByte();
3441
if (fileSnapshotKind == 1) {
3542
snapshots.put(key, new DefaultFileCollectionSnapshotter.DirSnapshot());

‎subprojects/core/src/main/groovy/org/gradle/api/internal/changedetection/state/OutputFilesCollectionSnapshotter.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package org.gradle.api.internal.changedetection.state;
1818

1919
import org.gradle.api.file.FileCollection;
20+
import org.gradle.api.internal.cache.StringInterner;
2021
import org.gradle.cache.PersistentIndexedCache;
2122
import org.gradle.internal.id.IdGenerator;
2223
import org.gradle.internal.serialize.DefaultSerializerRegistry;
@@ -45,19 +46,21 @@ public class OutputFilesCollectionSnapshotter implements FileCollectionSnapshott
4546
private final IdGenerator<Long> idGenerator;
4647
private final TaskArtifactStateCacheAccess cacheAccess;
4748
private final PersistentIndexedCache<String, Long> dirIdentifierCache;
49+
private final StringInterner stringInterner;
4850

4951
public OutputFilesCollectionSnapshotter(FileCollectionSnapshotter snapshotter, IdGenerator<Long> idGenerator,
50-
TaskArtifactStateCacheAccess cacheAccess) {
52+
TaskArtifactStateCacheAccess cacheAccess, StringInterner stringInterner) {
5153
this.snapshotter = snapshotter;
5254
this.idGenerator = idGenerator;
5355
this.cacheAccess = cacheAccess;
5456
dirIdentifierCache = cacheAccess.createCache("outputFileStates", String.class, new LongSerializer());
57+
this.stringInterner = stringInterner;
5558
}
5659

5760
public void registerSerializers(SerializerRegistry<FileCollectionSnapshot> registry) {
5861
DefaultSerializerRegistry<FileCollectionSnapshot> nested = new DefaultSerializerRegistry<FileCollectionSnapshot>();
5962
snapshotter.registerSerializers(nested);
60-
registry.register(OutputFilesSnapshot.class, new OutputFilesSnapshotSerializer(nested.build()));
63+
registry.register(OutputFilesSnapshot.class, new OutputFilesSnapshotSerializer(nested.build(), stringInterner));
6164
}
6265

6366
public FileCollectionSnapshot emptySnapshot() {
@@ -71,17 +74,18 @@ public OutputFilesSnapshot snapshot(final FileCollection files) {
7174
public void run() {
7275
for (File file : theFiles) {
7376
Long dirId;
77+
final String absolutePath = stringInterner.intern(file.getAbsolutePath());
7478
if (file.exists()) {
75-
dirId = dirIdentifierCache.get(file.getAbsolutePath());
79+
dirId = dirIdentifierCache.get(absolutePath);
7680
if (dirId == null) {
7781
dirId = idGenerator.generateId();
78-
dirIdentifierCache.put(file.getAbsolutePath(), dirId);
82+
dirIdentifierCache.put(absolutePath, dirId);
7983
}
8084
} else {
81-
dirIdentifierCache.remove(file.getAbsolutePath());
85+
dirIdentifierCache.remove(absolutePath);
8286
dirId = null;
8387
}
84-
snapshotDirIds.put(file.getAbsolutePath(), dirId);
88+
snapshotDirIds.put(absolutePath, dirId);
8589
}
8690

8791
}

‎subprojects/core/src/main/groovy/org/gradle/api/internal/changedetection/state/OutputFilesSnapshotSerializer.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package org.gradle.api.internal.changedetection.state;
1818

19+
import org.gradle.api.internal.cache.StringInterner;
1920
import org.gradle.internal.serialize.Decoder;
2021
import org.gradle.internal.serialize.Encoder;
2122
import org.gradle.internal.serialize.Serializer;
@@ -25,16 +26,18 @@
2526

2627
class OutputFilesSnapshotSerializer implements Serializer<OutputFilesCollectionSnapshotter.OutputFilesSnapshot> {
2728
private final Serializer<FileCollectionSnapshot> serializer;
29+
private final StringInterner stringInterner;
2830

29-
public OutputFilesSnapshotSerializer(Serializer<FileCollectionSnapshot> serializer) {
31+
public OutputFilesSnapshotSerializer(Serializer<FileCollectionSnapshot> serializer, StringInterner stringInterner) {
3032
this.serializer = serializer;
33+
this.stringInterner = stringInterner;
3134
}
3235

3336
public OutputFilesCollectionSnapshotter.OutputFilesSnapshot read(Decoder decoder) throws Exception {
3437
Map<String, Long> rootFileIds = new HashMap<String, Long>();
3538
int rootFileIdsCount = decoder.readSmallInt();
3639
for (int i = 0; i < rootFileIdsCount; i++) {
37-
String key = decoder.readString();
40+
String key = stringInterner.intern(decoder.readString());
3841
boolean notNull = decoder.readBoolean();
3942
Long value = notNull? decoder.readLong() : null;
4043
rootFileIds.put(key, value);

‎subprojects/core/src/main/groovy/org/gradle/internal/service/scopes/GlobalScopeServices.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import com.google.common.collect.Iterables;
2020
import org.gradle.StartParameter;
2121
import org.gradle.api.internal.*;
22+
import org.gradle.api.internal.cache.StringInterner;
2223
import org.gradle.api.internal.changedetection.state.CachingFileSnapshotter;
2324
import org.gradle.api.internal.changedetection.state.InMemoryTaskArtifactCache;
2425
import org.gradle.api.internal.classpath.*;
@@ -206,9 +207,9 @@ ModelRuleExtractor createModelRuleInspector(ServiceRegistry services, ModelSchem
206207
return new ModelRuleExtractor(Iterables.concat(coreExtractors, extractors));
207208
}
208209

209-
ClassPathSnapshotter createClassPathSnapshotter(GradleBuildEnvironment environment) {
210+
ClassPathSnapshotter createClassPathSnapshotter(GradleBuildEnvironment environment, StringInterner stringInterner) {
210211
if (environment.isLongLivingProcess()) {
211-
CachingFileSnapshotter fileSnapshotter = new CachingFileSnapshotter(new DefaultHasher(), new NonThreadsafeInMemoryStore());
212+
CachingFileSnapshotter fileSnapshotter = new CachingFileSnapshotter(new DefaultHasher(), new NonThreadsafeInMemoryStore(), stringInterner);
212213
return new HashClassPathSnapshotter(fileSnapshotter);
213214
} else {
214215
return new FileClassPathSnapshotter();
@@ -258,4 +259,7 @@ FileWatcherFactory createFileWatcherFactory(ExecutorFactory executorFactory) {
258259
return new DefaultFileWatcherFactory(executorFactory);
259260
}
260261

262+
StringInterner createStringInterner() {
263+
return new StringInterner();
264+
}
261265
}

‎subprojects/core/src/main/groovy/org/gradle/internal/service/scopes/TaskExecutionServices.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.gradle.StartParameter;
1919
import org.gradle.api.execution.TaskActionListener;
2020
import org.gradle.api.execution.internal.TaskInputsListener;
21+
import org.gradle.api.internal.cache.StringInterner;
2122
import org.gradle.api.internal.changedetection.TaskArtifactStateRepository;
2223
import org.gradle.api.internal.changedetection.changes.DefaultTaskArtifactStateRepository;
2324
import org.gradle.api.internal.changedetection.changes.ShortCircuitTaskArtifactStateRepository;
@@ -80,14 +81,14 @@ TaskArtifactStateCacheAccess createCacheAccess(Gradle gradle, CacheRepository ca
8081
return new DefaultTaskArtifactStateCacheAccess(gradle, cacheRepository, decorator);
8182
}
8283

83-
FileTreeElementSnapshotter createFileSnapshotter(TaskArtifactStateCacheAccess cacheAccess) {
84-
return new CachingFileSnapshotter(new DefaultHasher(), cacheAccess);
84+
FileTreeElementSnapshotter createFileSnapshotter(TaskArtifactStateCacheAccess cacheAccess, StringInterner stringInterner) {
85+
return new CachingFileSnapshotter(new DefaultHasher(), cacheAccess, stringInterner);
8586
}
8687

87-
TaskArtifactStateRepository createTaskArtifactStateRepository(Instantiator instantiator, TaskArtifactStateCacheAccess cacheAccess, StartParameter startParameter, FileTreeElementSnapshotter fileSnapshotter) {
88-
FileCollectionSnapshotter fileCollectionSnapshotter = new DefaultFileCollectionSnapshotter(fileSnapshotter, cacheAccess);
88+
TaskArtifactStateRepository createTaskArtifactStateRepository(Instantiator instantiator, TaskArtifactStateCacheAccess cacheAccess, StartParameter startParameter, FileTreeElementSnapshotter fileSnapshotter, StringInterner stringInterner) {
89+
FileCollectionSnapshotter fileCollectionSnapshotter = new DefaultFileCollectionSnapshotter(fileSnapshotter, cacheAccess, stringInterner);
8990

90-
FileCollectionSnapshotter outputFilesSnapshotter = new OutputFilesCollectionSnapshotter(fileCollectionSnapshotter, new RandomLongIdGenerator(), cacheAccess);
91+
FileCollectionSnapshotter outputFilesSnapshotter = new OutputFilesCollectionSnapshotter(fileCollectionSnapshotter, new RandomLongIdGenerator(), cacheAccess, stringInterner);
9192

9293
SerializerRegistry<FileCollectionSnapshot> serializerRegistry = new DefaultSerializerRegistry<FileCollectionSnapshot>();
9394
fileCollectionSnapshotter.registerSerializers(serializerRegistry);
@@ -96,7 +97,8 @@ TaskArtifactStateRepository createTaskArtifactStateRepository(Instantiator insta
9697
TaskHistoryRepository taskHistoryRepository = new CacheBackedTaskHistoryRepository(cacheAccess,
9798
new CacheBackedFileSnapshotRepository(cacheAccess,
9899
serializerRegistry.build(),
99-
new RandomLongIdGenerator()));
100+
new RandomLongIdGenerator()),
101+
stringInterner);
100102

101103
return new ShortCircuitTaskArtifactStateRepository(
102104
startParameter,

0 commit comments

Comments
 (0)