Skip to content

Commit e5e1107

Browse files
BenWhiteheadBenWhitehead
committed
test: replace ping bucket with per-run generated bucket
* Update tests to create objects for their tests * Add try-with around BlobDescriptor usages to ensure cleanup
1 parent 79bea2c commit e5e1107

File tree

1 file changed

+84
-52
lines changed

1 file changed

+84
-52
lines changed

‎google-cloud-storage/src/test/java/com/google/cloud/storage/ITBlobDescriptorTest.java‎

Lines changed: 84 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,33 @@
1616

1717
package com.google.cloud.storage;
1818

19+
import static com.google.cloud.storage.ByteSizeConstants._1MiB;
1920
import static com.google.cloud.storage.ByteSizeConstants._2MiB;
2021
import static com.google.cloud.storage.TestUtils.assertAll;
22+
import static com.google.cloud.storage.TestUtils.xxd;
2123
import static com.google.common.truth.Truth.assertThat;
2224
import static org.junit.Assert.assertThrows;
2325

2426
import com.google.api.core.ApiFuture;
2527
import com.google.api.core.ApiFutures;
2628
import com.google.cloud.storage.BlobDescriptor.ZeroCopySupport.DisposableByteString;
2729
import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown;
30+
import com.google.cloud.storage.Storage.BlobTargetOption;
2831
import com.google.cloud.storage.TransportCompatibility.Transport;
32+
import com.google.cloud.storage.it.ChecksummedTestContent;
2933
import com.google.cloud.storage.it.runner.StorageITRunner;
3034
import com.google.cloud.storage.it.runner.annotations.Backend;
3135
import com.google.cloud.storage.it.runner.annotations.Inject;
3236
import com.google.cloud.storage.it.runner.annotations.SingleBackend;
3337
import com.google.cloud.storage.it.runner.annotations.StorageFixture;
38+
import com.google.cloud.storage.it.runner.registry.Generator;
39+
import com.google.cloud.storage.it.runner.registry.ObjectsFixture;
40+
import com.google.cloud.storage.it.runner.registry.ObjectsFixture.ObjectAndContent;
3441
import com.google.common.base.Stopwatch;
3542
import com.google.common.hash.Hasher;
3643
import com.google.common.hash.Hashing;
3744
import com.google.protobuf.ByteString;
45+
import java.io.IOException;
3846
import java.nio.ByteBuffer;
3947
import java.util.List;
4048
import java.util.concurrent.ExecutionException;
@@ -55,72 +63,99 @@ public final class ITBlobDescriptorTest {
5563
@StorageFixture(Transport.GRPC)
5664
public Storage storage;
5765

66+
@Inject public BucketInfo bucket;
67+
68+
@Inject public Generator generator;
69+
70+
@Inject public ObjectsFixture objectsFixture;
71+
5872
@Test
59-
public void bytes() throws ExecutionException, InterruptedException, TimeoutException {
60-
BlobId blobId = BlobId.of("ping", "someobject");
73+
public void bytes()
74+
throws ExecutionException, InterruptedException, TimeoutException, IOException {
75+
ObjectAndContent obj512KiB = objectsFixture.getObj512KiB();
76+
byte[] expected = obj512KiB.getContent().getBytes(_512KiB - 13);
77+
BlobId blobId = obj512KiB.getInfo().getBlobId();
78+
79+
try (BlobDescriptor blobDescriptor =
80+
storage.getBlobDescriptor(blobId).get(30, TimeUnit.SECONDS)) {
6181

62-
BlobDescriptor blobDescriptor = storage.getBlobDescriptor(blobId).get(30, TimeUnit.SECONDS);
82+
BlobInfo info1 = blobDescriptor.getBlobInfo();
83+
assertThat(info1).isNotNull();
6384

64-
BlobInfo info1 = blobDescriptor.getBlobInfo();
65-
assertThat(info1).isNotNull();
85+
ApiFuture<byte[]> futureRead1Bytes =
86+
blobDescriptor.readRangeAsBytes(ByteRangeSpec.relativeLength(_512KiB - 13L, 13L));
6687

67-
ApiFuture<byte[]> futureRead1Bytes =
68-
blobDescriptor.readRangeAsBytes(ByteRangeSpec.relativeLength(_512KiB - 13L, 13L));
88+
byte[] read1Bytes = futureRead1Bytes.get(30, TimeUnit.SECONDS);
89+
assertThat(read1Bytes.length).isEqualTo(13);
6990

70-
byte[] read1Bytes = futureRead1Bytes.get(30, TimeUnit.SECONDS);
71-
assertThat(read1Bytes.length).isEqualTo(13);
91+
assertThat(xxd(read1Bytes)).isEqualTo(xxd(expected));
92+
}
7293
}
7394

7495
@Test
7596
public void lotsOfBytes() throws Exception {
76-
BlobId blobId = BlobId.of("ping", "someobject");
97+
ChecksummedTestContent testContent =
98+
ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(512 * _1MiB));
99+
100+
BlobInfo gen1 =
101+
storage.create(
102+
BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(),
103+
testContent.getBytes(),
104+
BlobTargetOption.doesNotExist());
105+
BlobId blobId = gen1.getBlobId();
77106
for (int j = 0; j < 2; j++) {
78107

79-
BlobDescriptor blobDescriptor = storage.getBlobDescriptor(blobId).get(30, TimeUnit.SECONDS);
80-
81108
Stopwatch sw = Stopwatch.createStarted();
82-
int numRangesToRead = 256;
83-
List<ApiFuture<byte[]>> futures =
84-
LongStream.range(0, numRangesToRead)
85-
.mapToObj(i -> ByteRangeSpec.relativeLength(i * _2MiB, (long) _2MiB))
86-
.map(blobDescriptor::readRangeAsBytes)
87-
.collect(Collectors.toList());
88-
89-
ApiFuture<List<byte[]>> listApiFuture = ApiFutures.allAsList(futures);
90-
91-
List<byte[]> ranges = listApiFuture.get(5, TimeUnit.SECONDS);
92-
Stopwatch stop = sw.stop();
93-
System.out.println(stop.elapsed(TimeUnit.MILLISECONDS));
94-
Hasher hasher = Hashing.crc32c().newHasher();
95-
long length = 0;
96-
for (byte[] range : ranges) {
97-
hasher.putBytes(range);
98-
length += range.length;
99-
}
100-
final long finalLength = length;
101-
102-
assertAll(
103-
() -> {
104-
Hasher xHasher = Hashing.crc32c().newHasher();
105-
long numBytes = numRangesToRead * _2MiB;
106-
for (long l = 0; l < numBytes; l++) {
107-
xHasher.putByte((byte) 'x');
108-
}
109+
try (BlobDescriptor blobDescriptor =
110+
storage.getBlobDescriptor(blobId).get(30, TimeUnit.SECONDS)) {
111+
112+
int numRangesToRead = 256;
113+
List<ApiFuture<byte[]>> futures =
114+
LongStream.range(0, numRangesToRead)
115+
.mapToObj(i -> ByteRangeSpec.relativeLength(i * _2MiB, (long) _2MiB))
116+
.map(blobDescriptor::readRangeAsBytes)
117+
.collect(Collectors.toList());
118+
119+
ApiFuture<List<byte[]>> listApiFuture = ApiFutures.allAsList(futures);
109120

110-
Crc32cLengthKnown expectedCrc32c = Crc32cValue.of(xHasher.hash().asInt(), numBytes);
111-
Crc32cLengthKnown actualCrc32c = Crc32cValue.of(hasher.hash().asInt(), finalLength);
121+
List<byte[]> ranges = listApiFuture.get(5, TimeUnit.SECONDS);
122+
Stopwatch stop = sw.stop();
123+
System.out.println(stop.elapsed(TimeUnit.MILLISECONDS));
124+
Hasher hasher = Hashing.crc32c().newHasher();
125+
long length = 0;
126+
for (byte[] range : ranges) {
127+
hasher.putBytes(range);
128+
length += range.length;
129+
}
130+
final long finalLength = length;
131+
132+
assertAll(
133+
() -> {
134+
Crc32cLengthKnown expectedCrc32c =
135+
Crc32cValue.of(testContent.getCrc32c(), testContent.getBytes().length);
136+
Crc32cLengthKnown actualCrc32c = Crc32cValue.of(hasher.hash().asInt(), finalLength);
112137

113-
assertThat(actualCrc32c).isEqualTo(expectedCrc32c);
114-
},
115-
() -> assertThat(finalLength).isEqualTo(numRangesToRead * _2MiB));
138+
assertThat(actualCrc32c).isEqualTo(expectedCrc32c);
139+
},
140+
() -> assertThat(finalLength).isEqualTo(numRangesToRead * _2MiB));
141+
}
116142
}
117143
}
118144

119145
@Test
120146
public void readRangeAsByteString() throws Exception {
121-
BlobId blobId = BlobId.of("ping", "someobject");
147+
ChecksummedTestContent testContent =
148+
ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(512 * _1MiB));
149+
150+
BlobInfo gen1 =
151+
storage.create(
152+
BlobInfo.newBuilder(bucket, generator.randomObjectName()).build(),
153+
testContent.getBytes(),
154+
BlobTargetOption.doesNotExist());
155+
BlobId blobId = gen1.getBlobId();
122156
for (int j = 0; j < 2; j++) {
123157

158+
Stopwatch sw = Stopwatch.createStarted();
124159
try (BlobDescriptorImpl blobDescriptor =
125160
(BlobDescriptorImpl) storage.getBlobDescriptor(blobId).get(2, TimeUnit.SECONDS)) {
126161

@@ -134,6 +169,8 @@ public void readRangeAsByteString() throws Exception {
134169
ApiFuture<List<DisposableByteString>> listApiFuture = ApiFutures.allAsList(futures);
135170

136171
List<DisposableByteString> ranges = listApiFuture.get(5, TimeUnit.SECONDS);
172+
Stopwatch stop = sw.stop();
173+
System.out.println(stop.elapsed(TimeUnit.MILLISECONDS));
137174
Hasher hasher = Hashing.crc32c().newHasher();
138175
long length = 0;
139176
for (DisposableByteString range : ranges) {
@@ -149,13 +186,8 @@ public void readRangeAsByteString() throws Exception {
149186

150187
assertAll(
151188
() -> {
152-
Hasher xHasher = Hashing.crc32c().newHasher();
153-
long numBytes = numRangesToRead * _2MiB;
154-
for (long l = 0; l < numBytes; l++) {
155-
xHasher.putByte((byte) 'x');
156-
}
157-
158-
Crc32cLengthKnown expectedCrc32c = Crc32cValue.of(xHasher.hash().asInt(), numBytes);
189+
Crc32cLengthKnown expectedCrc32c =
190+
Crc32cValue.of(testContent.getCrc32c(), testContent.getBytes().length);
159191
Crc32cLengthKnown actualCrc32c = Crc32cValue.of(hasher.hash().asInt(), finalLength);
160192

161193
assertThat(actualCrc32c).isEqualTo(expectedCrc32c);

0 commit comments

Comments
 (0)