Skip to content

Commit d98ecc7

Browse files
BenWhiteheadBenWhitehead
authored
chore: update LinearExponentialRangeSpecFunction to reset its spec length if the provided previous range is not sequential
If a "seek" happens, reset the spec length to the min range size. Update ITObjectReadSessionTest#seekable to use a smaller size object to avoid timeouts with testbench.
1 parent 6858a9d commit d98ecc7

File tree

3 files changed

+96
-21
lines changed

3 files changed

+96
-21
lines changed

‎google-cloud-storage/src/main/java/com/google/cloud/storage/LinearExponentialRangeSpecFunction.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ public RangeSpec apply(long offset, RangeSpec prev) {
6464
long limit;
6565
if (maybeLimit.isPresent()) {
6666
limit = maybeLimit.getAsLong();
67+
68+
long expectedOffset = prev.begin() + limit;
69+
if (offset != expectedOffset) {
70+
return RangeSpec.of(offset, initialRangeSize);
71+
}
72+
6773
} else {
6874
limit = Long.MAX_VALUE;
6975
}

‎google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionTest.java‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.google.api.core.ApiFuture;
2727
import com.google.api.core.ApiFutures;
2828
import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown;
29+
import com.google.cloud.storage.RangeProjectionConfigs.SeekableChannelConfig;
2930
import com.google.cloud.storage.Storage.BlobTargetOption;
3031
import com.google.cloud.storage.TransportCompatibility.Transport;
3132
import com.google.cloud.storage.ZeroCopySupport.DisposableByteString;
@@ -256,25 +257,31 @@ public void readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404() {
256257
@Test
257258
public void seekable() throws Exception {
258259
ChecksummedTestContent testContent =
259-
ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(64 * _1MiB));
260+
ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(16 * _1MiB));
261+
262+
SeekableChannelConfig config =
263+
RangeProjectionConfigs.asSeekableChannel()
264+
.withRangeSpecFunction(
265+
RangeSpecFunction.linearExponential()
266+
.withMinRangeSize(_1MiB)
267+
.withRangeSizeScalar(4.0));
260268

261269
BlobInfo gen1 = create(testContent);
262270
BlobId blobId = gen1.getBlobId();
263271
ByteBuffer buf = ByteBuffer.allocate(2 * 1024 * 1024);
264-
for (int j = 0; j < 2; j++) {
272+
for (int j = 0; j < 1; j++) {
265273

266-
try (BlobReadSession session = storage.blobReadSession(blobId).get(2, TimeUnit.SECONDS)) {
274+
try (BlobReadSession session = storage.blobReadSession(blobId).get(30, TimeUnit.SECONDS)) {
267275
CountingOutputStream countingOutputStream =
268276
new CountingOutputStream(ByteStreams.nullOutputStream());
269277
long copy1;
270278
long copy2;
271279
long copy3;
272-
try (SeekableByteChannel seekable =
273-
session.readAs(RangeProjectionConfigs.asSeekableChannel());
280+
try (SeekableByteChannel seekable = session.readAs(config);
274281
WritableByteChannel w = Channels.newChannel(countingOutputStream)) {
275282
copy1 = Buffers.copyUsingBuffer(buf, seekable, w);
276283

277-
seekable.position(32 * _1MiB);
284+
seekable.position(8 * _1MiB);
278285
copy2 = Buffers.copyUsingBuffer(buf, seekable, w);
279286

280287
seekable.position(0);
@@ -286,10 +293,10 @@ public void seekable() throws Exception {
286293
long finalCopy2 = copy2;
287294
long finalCopy3 = copy3;
288295
assertAll(
289-
() -> assertThat(totalRead).isEqualTo((64 + 32 + 64) * _1MiB),
290-
() -> assertThat(finalCopy1).isEqualTo(64 * _1MiB),
291-
() -> assertThat(finalCopy2).isEqualTo(32 * _1MiB),
292-
() -> assertThat(finalCopy3).isEqualTo(64 * _1MiB));
296+
() -> assertThat(totalRead).isEqualTo((16 + 8 + 16) * _1MiB),
297+
() -> assertThat(finalCopy1).isEqualTo(16 * _1MiB),
298+
() -> assertThat(finalCopy2).isEqualTo(8 * _1MiB),
299+
() -> assertThat(finalCopy3).isEqualTo(16 * _1MiB));
293300
}
294301
}
295302
}

‎google-cloud-storage/src/test/java/com/google/cloud/storage/RangeSpecFunctionTest.java‎

Lines changed: 73 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@
1818

1919
import static com.google.common.truth.Truth.assertThat;
2020

21-
import net.jqwik.api.Example;
21+
import org.junit.Test;
2222

2323
public final class RangeSpecFunctionTest {
2424
private static final long KiB = 1024;
2525
private static final long MiB = 1024 * KiB;
2626

2727
@SuppressWarnings("OptionalGetWithoutIsPresent")
28-
@Example
28+
@Test
2929
public void linearExponential_withMaxLimit() {
3030
RangeSpecFunction e =
3131
RangeSpecFunction.linearExponential()
@@ -37,23 +37,85 @@ public void linearExponential_withMaxLimit() {
3737

3838
apply = e.apply(0, apply);
3939
assertThat(apply).isEqualTo(RangeSpec.of(0, KiB));
40-
apply = e.apply(apply.limit().getAsLong(), apply);
40+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
4141
assertThat(apply.limit().getAsLong()).isEqualTo(4 * KiB);
42-
apply = e.apply(apply.limit().getAsLong(), apply);
42+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
4343
assertThat(apply.limit().getAsLong()).isEqualTo(16 * KiB);
44-
apply = e.apply(apply.limit().getAsLong(), apply);
44+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
4545
assertThat(apply.limit().getAsLong()).isEqualTo(64 * KiB);
46-
apply = e.apply(apply.limit().getAsLong(), apply);
46+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
4747
assertThat(apply.limit().getAsLong()).isEqualTo(256 * KiB);
48-
apply = e.apply(apply.limit().getAsLong(), apply);
48+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
4949
assertThat(apply.limit().getAsLong()).isEqualTo(MiB);
50-
apply = e.apply(apply.limit().getAsLong(), apply);
50+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
5151
assertThat(apply.limit().getAsLong()).isEqualTo(4 * MiB);
52-
apply = e.apply(apply.limit().getAsLong(), apply);
52+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
5353
assertThat(apply.limit().getAsLong()).isEqualTo(16 * MiB);
54-
apply = e.apply(apply.limit().getAsLong(), apply);
54+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
5555
assertThat(apply.limit().getAsLong()).isEqualTo(64 * MiB);
56-
apply = e.apply(apply.limit().getAsLong(), apply);
56+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
5757
assertThat(apply.limit().getAsLong()).isEqualTo(64 * MiB);
5858
}
59+
60+
@SuppressWarnings("OptionalGetWithoutIsPresent")
61+
@Test
62+
public void linearExponential_resetsIfNotSequential_forward() {
63+
RangeSpecFunction e =
64+
RangeSpecFunction.linearExponential().withMinRangeSize(KiB).withRangeSizeScalar(4.0);
65+
66+
RangeSpec apply = null;
67+
68+
apply = e.apply(0, apply);
69+
assertThat(apply).isEqualTo(RangeSpec.of(0, KiB));
70+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
71+
assertThat(apply.limit().getAsLong()).isEqualTo(4 * KiB);
72+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
73+
assertThat(apply.limit().getAsLong()).isEqualTo(16 * KiB);
74+
75+
apply = e.apply(apply.begin() + apply.limit().getAsLong() + 1, apply);
76+
assertThat(apply.limit().getAsLong()).isEqualTo(KiB);
77+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
78+
assertThat(apply.limit().getAsLong()).isEqualTo(4 * KiB);
79+
}
80+
81+
@SuppressWarnings("OptionalGetWithoutIsPresent")
82+
@Test
83+
public void linearExponential_resetsIfNotSequential_backward() {
84+
RangeSpecFunction e =
85+
RangeSpecFunction.linearExponential().withMinRangeSize(KiB).withRangeSizeScalar(4.0);
86+
87+
RangeSpec apply = null;
88+
89+
apply = e.apply(0, apply);
90+
assertThat(apply).isEqualTo(RangeSpec.of(0, KiB));
91+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
92+
assertThat(apply.limit().getAsLong()).isEqualTo(4 * KiB);
93+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
94+
assertThat(apply.limit().getAsLong()).isEqualTo(16 * KiB);
95+
96+
apply = e.apply(apply.begin() + apply.limit().getAsLong() - 1, apply);
97+
assertThat(apply.limit().getAsLong()).isEqualTo(KiB);
98+
apply = e.apply(apply.begin() + apply.limit().getAsLong(), apply);
99+
assertThat(apply.limit().getAsLong()).isEqualTo(4 * KiB);
100+
}
101+
102+
@Test
103+
public void linearExponential_resetsIfNotSequential() {
104+
RangeSpecFunction e =
105+
RangeSpecFunction.linearExponential().withMinRangeSize(1).withRangeSizeScalar(4.0);
106+
107+
RangeSpec apply = null;
108+
109+
apply = e.apply(0, apply);
110+
assertThat(apply).isEqualTo(RangeSpec.of(0, 1));
111+
apply = e.apply(1, apply);
112+
assertThat(apply).isEqualTo(RangeSpec.of(1, 4));
113+
apply = e.apply(5, apply);
114+
assertThat(apply).isEqualTo(RangeSpec.of(5, 16));
115+
116+
apply = e.apply(4, apply);
117+
assertThat(apply).isEqualTo(RangeSpec.of(4, 1));
118+
apply = e.apply(5, apply);
119+
assertThat(apply).isEqualTo(RangeSpec.of(5, 4));
120+
}
59121
}

0 commit comments

Comments
 (0)