Skip to content

Commit d49263b

Browse files
jshore1296tritone
jshore1296
and
tritone
authored
feat(storage): return file metadata on read (#11212)
* storage: Return file Metadata on file read * review comments * review comments * gofmt * Access metadata through function, rather than field Adding a map field to Attrs would have made the object uncomparable * Store metadata as a pointer to preserve API compatibility - keeps the Reader struct comparable --------- Co-authored-by: Chris Cotter <cjcotter@google.com>
1 parent c10a4bd commit d49263b

File tree

6 files changed

+202
-7
lines changed

6 files changed

+202
-7
lines changed

‎storage/client_test.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -795,6 +795,67 @@ func TestOpenReaderEmulated(t *testing.T) {
795795
})
796796
}
797797

798+
func TestOpenReaderEmulated_Metadata(t *testing.T) {
799+
transportClientTest(context.Background(), t, func(t *testing.T, ctx context.Context, project, bucket string, client storageClient) {
800+
// Populate test data.
801+
_, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{
802+
Name: bucket,
803+
}, nil)
804+
if err != nil {
805+
t.Fatalf("client.CreateBucket: %v", err)
806+
}
807+
prefix := time.Now().Nanosecond()
808+
want := &ObjectAttrs{
809+
Bucket: bucket,
810+
Name: fmt.Sprintf("%d-object-%d", prefix, time.Now().Nanosecond()),
811+
}
812+
w := veneerClient.Bucket(bucket).Object(want.Name).NewWriter(ctx)
813+
if _, err := w.Write(randomBytesToWrite); err != nil {
814+
t.Fatalf("failed to populate test data: %v", err)
815+
}
816+
if err := w.Close(); err != nil {
817+
t.Fatalf("closing object: %v", err)
818+
}
819+
if _, err := veneerClient.Bucket(bucket).Object(want.Name).Update(ctx, ObjectAttrsToUpdate{
820+
Metadata: map[string]string{
821+
"Custom-Key": "custom-value",
822+
"Some-Other-Key": "some-other-value",
823+
},
824+
}); err != nil {
825+
t.Fatalf("failed to update test object: %v", err)
826+
}
827+
828+
params := &newRangeReaderParams{
829+
bucket: bucket,
830+
object: want.Name,
831+
gen: defaultGen,
832+
offset: 0,
833+
length: -1,
834+
}
835+
r, err := client.NewRangeReader(ctx, params)
836+
if err != nil {
837+
t.Fatalf("opening reading: %v", err)
838+
}
839+
wantLen := len(randomBytesToWrite)
840+
got := make([]byte, wantLen)
841+
n, err := r.Read(got)
842+
if n != wantLen {
843+
t.Fatalf("expected to read %d bytes, but got %d", wantLen, n)
844+
}
845+
if diff := cmp.Diff(got, randomBytesToWrite); diff != "" {
846+
t.Fatalf("Read: got(-),want(+):\n%s", diff)
847+
}
848+
expectedMetadata := map[string]string{
849+
"Custom-Key": "custom-value",
850+
"Some-Other-Key": "some-other-value",
851+
}
852+
if diff := cmp.Diff(r.Metadata(), expectedMetadata); diff != "" {
853+
t.Fatalf("Object Metadata: got(-),want(+):\n%s", diff)
854+
}
855+
856+
})
857+
}
858+
798859
func TestOpenWriterEmulated(t *testing.T) {
799860
transportClientTest(context.Background(), t, func(t *testing.T, ctx context.Context, project, bucket string, client storageClient) {
800861
// Populate test data.

‎storage/grpc_client.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1125,6 +1125,7 @@ func (c *grpcStorageClient) NewRangeReader(ctx context.Context, params *newRange
11251125
wantCRC = checksums.GetCrc32C()
11261126
}
11271127

1128+
metadata := obj.GetMetadata()
11281129
r = &Reader{
11291130
Attrs: ReaderObjectAttrs{
11301131
Size: size,
@@ -1136,6 +1137,7 @@ func (c *grpcStorageClient) NewRangeReader(ctx context.Context, params *newRange
11361137
Generation: obj.GetGeneration(),
11371138
CRC32C: wantCRC,
11381139
},
1140+
objectMetadata: &metadata,
11391141
reader: &gRPCReader{
11401142
stream: res.stream,
11411143
reopen: reopen,

‎storage/http_client.go‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1523,6 +1523,14 @@ func parseReadResponse(res *http.Response, params *newRangeReaderParams, reopen
15231523
}
15241524
}
15251525

1526+
metadata := map[string]string{}
1527+
for key, values := range res.Header {
1528+
if len(values) > 0 && strings.HasPrefix(key, "X-Goog-Meta-") {
1529+
key := key[len("X-Goog-Meta-"):]
1530+
metadata[key] = values[0]
1531+
}
1532+
}
1533+
15261534
attrs := ReaderObjectAttrs{
15271535
Size: size,
15281536
ContentType: res.Header.Get("Content-Type"),
@@ -1536,10 +1544,11 @@ func parseReadResponse(res *http.Response, params *newRangeReaderParams, reopen
15361544
Decompressed: res.Uncompressed || uncompressedByServer(res),
15371545
}
15381546
return &Reader{
1539-
Attrs: attrs,
1540-
size: size,
1541-
remain: remain,
1542-
checkCRC: checkCRC,
1547+
Attrs: attrs,
1548+
objectMetadata: &metadata,
1549+
size: size,
1550+
remain: remain,
1551+
checkCRC: checkCRC,
15431552
reader: &httpReader{
15441553
reopen: reopen,
15451554
body: body,

‎storage/integration_test.go‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5044,8 +5044,48 @@ func TestIntegration_ReaderAttrs(t *testing.T) {
50445044
Metageneration: attrs.Metageneration,
50455045
CRC32C: crc32c(c),
50465046
}
5047-
if got != want {
5048-
t.Fatalf("got\t%v,\nwanted\t%v", got, want)
5047+
if diff := cmp.Diff(got, want); diff != "" {
5048+
t.Fatalf("diff got vs want: %v", diff)
5049+
}
5050+
})
5051+
}
5052+
5053+
func TestIntegration_ReaderAttrs_Metadata(t *testing.T) {
5054+
multiTransportTest(skipJSONReads(context.Background(), "metadata on read not supported on JSON api"), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
5055+
bkt := client.Bucket(bucket)
5056+
5057+
const defaultType = "text/plain"
5058+
o := bkt.Object("reader-attrs-metadata-obj")
5059+
c := randomContents()
5060+
if err := writeObject(ctx, o, defaultType, c); err != nil {
5061+
t.Errorf("Write for %v failed with %v", o.ObjectName(), err)
5062+
}
5063+
t.Cleanup(func() {
5064+
if err := o.Delete(ctx); err != nil {
5065+
log.Printf("failed to delete test object: %v", err)
5066+
}
5067+
})
5068+
5069+
oa, err := o.Update(ctx, ObjectAttrsToUpdate{Metadata: map[string]string{"Custom-Key": "custom-value", "Other-Key": "other-value"}})
5070+
if err != nil {
5071+
t.Fatal(err)
5072+
}
5073+
_ = oa
5074+
5075+
o = o.Generation(oa.Generation)
5076+
rc, err := o.NewReader(ctx)
5077+
if err != nil {
5078+
t.Fatal(err)
5079+
}
5080+
5081+
got := rc.Metadata()
5082+
want := map[string]string{
5083+
"Custom-Key": "custom-value",
5084+
"Other-Key": "other-value",
5085+
}
5086+
5087+
if diff := cmp.Diff(got, want); diff != "" {
5088+
t.Fatalf("diff got vs want: %v", diff)
50495089
}
50505090
})
50515091
}

‎storage/reader.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,9 @@ var emptyBody = ioutil.NopCloser(strings.NewReader(""))
222222
// the stored CRC, returning an error from Read if there is a mismatch. This integrity check
223223
// is skipped if transcoding occurs. See https://cloud.google.com/storage/docs/transcoding.
224224
type Reader struct {
225-
Attrs ReaderObjectAttrs
225+
Attrs ReaderObjectAttrs
226+
objectMetadata *map[string]string
227+
226228
seen, remain, size int64
227229
checkCRC bool // Did we check the CRC? This is now only used by tests.
228230

@@ -298,3 +300,16 @@ func (r *Reader) CacheControl() string {
298300
func (r *Reader) LastModified() (time.Time, error) {
299301
return r.Attrs.LastModified, nil
300302
}
303+
304+
// Metadata returns user-provided metadata, in key/value pairs.
305+
//
306+
// It can be nil if no metadata is present, or if the client uses the JSON
307+
// API for downloads. Only the XML and gRPC APIs support getting
308+
// custom metadata via the Reader; for JSON make a separate call to
309+
// ObjectHandle.Attrs.
310+
func (r *Reader) Metadata() map[string]string {
311+
if r.objectMetadata != nil {
312+
return *r.objectMetadata
313+
}
314+
return nil
315+
}

‎storage/reader_test.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"strings"
3030
"testing"
3131

32+
"github.com/google/go-cmp/cmp"
3233
"google.golang.org/api/option"
3334
)
3435

@@ -380,6 +381,7 @@ func TestContentEncodingGzipWithReader(t *testing.T) {
380381
w.Header().Set("Etag", `"c50e3e41c9bc9df34e84c94ce073f928"`)
381382
w.Header().Set("X-Goog-Generation", "1587012235914578")
382383
w.Header().Set("X-Goog-MetaGeneration", "2")
384+
w.Header().Set("X-Goog-Meta-custom-metadata-key", "custom-metadata-value")
383385
w.Header().Set("X-Goog-Stored-Content-Encoding", "gzip")
384386
w.Header().Set("vary", "Accept-Encoding")
385387
w.Header().Set("x-goog-stored-content-length", "43")
@@ -470,6 +472,72 @@ func TestContentEncodingGzipWithReader(t *testing.T) {
470472
}, option.WithEndpoint(mockGCS.URL), option.WithoutAuthentication(), option.WithHTTPClient(whc))
471473
}
472474

475+
func TestMetadataParsingWithReader(t *testing.T) {
476+
bucketName := "my-bucket"
477+
objectName := "test"
478+
downloadObjectXMLurl := fmt.Sprintf("/%s/%s", bucketName, objectName)
479+
downloadObjectJSONurl := fmt.Sprintf("/b/%s/o/%s?alt=media&prettyPrint=false&projection=full", bucketName, objectName)
480+
481+
original := bytes.Repeat([]byte("a"), 4)
482+
mockGCS := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
483+
switch r.URL.String() {
484+
case downloadObjectXMLurl, downloadObjectJSONurl:
485+
// Serve back the file.
486+
w.Header().Set("Content-Type", "text/plain")
487+
w.Header().Set("Etag", `"c50e3e41c9bc9df34e84c94ce073f928"`)
488+
w.Header().Set("X-Goog-Generation", "1587012235914578")
489+
w.Header().Set("X-Goog-MetaGeneration", "2")
490+
w.Header().Set("X-Goog-Meta-custom-metadata-key", "custom-metadata-value")
491+
w.Header().Set("vary", "Accept-Encoding")
492+
w.Header().Set("x-goog-stored-content-length", "4")
493+
w.Header().Set("x-goog-hash", "crc32c=pYIWwQ==")
494+
w.Header().Set("x-goog-hash", "md5=xQ4+Qcm8nfNOhMlM4HP5KA==")
495+
w.Header().Set("x-goog-storage-class", "STANDARD")
496+
w.Write(original)
497+
default:
498+
fmt.Fprintf(w, "unrecognized URL %s", r.URL)
499+
}
500+
}))
501+
mockGCS.EnableHTTP2 = true
502+
mockGCS.StartTLS()
503+
defer mockGCS.Close()
504+
505+
ctx := context.Background()
506+
hc := mockGCS.Client()
507+
ux, _ := url.Parse(mockGCS.URL)
508+
hc.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify = true
509+
wrt := &alwaysToTargetURLRoundTripper{
510+
destURL: ux,
511+
hc: hc,
512+
}
513+
514+
whc := &http.Client{Transport: wrt}
515+
516+
multiReaderTest(ctx, t, func(t *testing.T, c *Client) {
517+
obj := c.Bucket(bucketName).Object(objectName)
518+
rd, err := obj.NewReader(ctx)
519+
if err != nil {
520+
t.Fatal(err)
521+
}
522+
defer rd.Close()
523+
524+
expectedMetadata := map[string]string{
525+
"Custom-Metadata-Key": "custom-metadata-value",
526+
}
527+
if diff := cmp.Diff(rd.Metadata(), expectedMetadata); diff != "" {
528+
t.Fatalf("metadata mismatch diff got vs want: %v", diff)
529+
}
530+
531+
got, err := ioutil.ReadAll(rd)
532+
if err != nil {
533+
t.Fatal(err)
534+
}
535+
if g, w := got, original; !bytes.Equal(g, w) {
536+
t.Fatalf("Response mismatch\nGot:\n%q\n\nWant:\n%q", g, w)
537+
}
538+
}, option.WithEndpoint(mockGCS.URL), option.WithoutAuthentication(), option.WithHTTPClient(whc))
539+
}
540+
473541
// alwaysToTargetURLRoundTripper ensures that every single request
474542
// is routed to a target destination. Some requests within the storage
475543
// client by-pass using the provided HTTP client, hence this enforcemenet.

0 commit comments

Comments
 (0)