Skip to content

Commit 6ac92e8

Browse files
jonjohnsonjrjonjohnsonjr
authored
Refactor fetcher, writer, and progress (#1625)
This will allow reuse across a repository. One major difference is that keychains are no longer resolved within the option execution, but lazily during fetcher/writer construction, which enables us to create options without knowing the repo.
1 parent 249d7e1 commit 6ac92e8

File tree

15 files changed

+179
-185
lines changed

15 files changed

+179
-185
lines changed

‎pkg/v1/remote/catalog.go‎

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -31,32 +31,27 @@ type catalog struct {
3131

3232
// CatalogPage calls /_catalog, returning the list of repositories on the registry.
3333
func CatalogPage(target name.Registry, last string, n int, options ...Option) ([]string, error) {
34-
o, err := makeOptions(target, options...)
34+
o, err := makeOptions(options...)
3535
if err != nil {
3636
return nil, err
3737
}
38-
39-
scopes := []string{target.Scope(transport.PullScope)}
40-
tr, err := transport.NewWithContext(o.context, target, o.auth, o.transport, scopes)
38+
f, err := makeFetcher(o.context, target, o)
4139
if err != nil {
4240
return nil, err
4341
}
4442

45-
query := fmt.Sprintf("last=%s&n=%d", url.QueryEscape(last), n)
46-
4743
uri := url.URL{
4844
Scheme: target.Scheme(),
4945
Host: target.RegistryStr(),
5046
Path: "/v2/_catalog",
51-
RawQuery: query,
47+
RawQuery: fmt.Sprintf("last=%s&n=%d", url.QueryEscape(last), n),
5248
}
5349

54-
client := http.Client{Transport: tr}
5550
req, err := http.NewRequest(http.MethodGet, uri.String(), nil)
5651
if err != nil {
5752
return nil, err
5853
}
59-
resp, err := client.Do(req.WithContext(o.context))
54+
resp, err := f.client.Do(req.WithContext(o.context))
6055
if err != nil {
6156
return nil, err
6257
}
@@ -76,13 +71,11 @@ func CatalogPage(target name.Registry, last string, n int, options ...Option) ([
7671

7772
// Catalog calls /_catalog, returning the list of repositories on the registry.
7873
func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]string, error) {
79-
o, err := makeOptions(target, options...)
74+
o, err := makeOptions(options...)
8075
if err != nil {
8176
return nil, err
8277
}
83-
84-
scopes := []string{target.Scope(transport.PullScope)}
85-
tr, err := transport.NewWithContext(o.context, target, o.auth, o.transport, scopes)
78+
f, err := makeFetcher(o.context, target, o)
8679
if err != nil {
8780
return nil, err
8881
}
@@ -92,13 +85,10 @@ func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]st
9285
Host: target.RegistryStr(),
9386
Path: "/v2/_catalog",
9487
}
95-
9688
if o.pageSize > 0 {
9789
uri.RawQuery = fmt.Sprintf("n=%d", o.pageSize)
9890
}
9991

100-
client := http.Client{Transport: tr}
101-
10292
// WithContext overrides the ctx passed directly.
10393
if o.context != context.Background() {
10494
ctx = o.context
@@ -123,7 +113,7 @@ func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]st
123113
}
124114
req = req.WithContext(ctx)
125115

126-
resp, err := client.Do(req)
116+
resp, err := f.client.Do(req)
127117
if err != nil {
128118
return nil, err
129119
}

‎pkg/v1/remote/delete.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,15 @@ import (
2525

2626
// Delete removes the specified image reference from the remote registry.
2727
func Delete(ref name.Reference, options ...Option) error {
28-
o, err := makeOptions(ref.Context(), options...)
28+
o, err := makeOptions(options...)
2929
if err != nil {
3030
return err
3131
}
32-
scopes := []string{ref.Scope(transport.DeleteScope)}
33-
tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, scopes)
32+
w, err := makeWriter(o.context, ref.Context(), nil, o)
3433
if err != nil {
3534
return err
3635
}
37-
c := &http.Client{Transport: tr}
36+
c := w.client
3837

3938
u := url.URL{
4039
Scheme: ref.Context().Registry.Scheme(),

‎pkg/v1/remote/descriptor.go‎

Lines changed: 66 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727

2828
"github.com/google/go-containerregistry/internal/redact"
2929
"github.com/google/go-containerregistry/internal/verify"
30+
"github.com/google/go-containerregistry/pkg/authn"
3031
"github.com/google/go-containerregistry/pkg/logs"
3132
"github.com/google/go-containerregistry/pkg/name"
3233
v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -59,6 +60,8 @@ func (e *ErrSchema1) Error() string {
5960
type Descriptor struct {
6061
fetcher
6162
v1.Descriptor
63+
64+
ref name.Reference
6265
Manifest []byte
6366

6467
// So we can share this implementation with Image.
@@ -100,36 +103,37 @@ func Head(ref name.Reference, options ...Option) (*v1.Descriptor, error) {
100103
acceptable = append(acceptable, acceptableImageMediaTypes...)
101104
acceptable = append(acceptable, acceptableIndexMediaTypes...)
102105

103-
o, err := makeOptions(ref.Context(), options...)
106+
o, err := makeOptions(options...)
104107
if err != nil {
105108
return nil, err
106109
}
107110

108-
f, err := makeFetcher(ref, o)
111+
f, err := makeFetcher(o.context, ref.Context(), o)
109112
if err != nil {
110113
return nil, err
111114
}
112115

113-
return f.headManifest(ref, acceptable)
116+
return f.headManifest(o.context, ref, acceptable)
114117
}
115118

116119
// Handle options and fetch the manifest with the acceptable MediaTypes in the
117120
// Accept header.
118121
func get(ref name.Reference, acceptable []types.MediaType, options ...Option) (*Descriptor, error) {
119-
o, err := makeOptions(ref.Context(), options...)
122+
o, err := makeOptions(options...)
120123
if err != nil {
121124
return nil, err
122125
}
123-
f, err := makeFetcher(ref, o)
126+
f, err := makeFetcher(o.context, ref.Context(), o)
124127
if err != nil {
125128
return nil, err
126129
}
127-
b, desc, err := f.fetchManifest(ref, acceptable)
130+
b, desc, err := f.fetchManifest(o.context, ref, acceptable)
128131
if err != nil {
129132
return nil, err
130133
}
131134
return &Descriptor{
132135
fetcher: *f,
136+
ref: ref,
133137
Manifest: b,
134138
Descriptor: *desc,
135139
platform: o.platform,
@@ -169,7 +173,7 @@ func (d *Descriptor) Image() (v1.Image, error) {
169173
}
170174
return &mountableImage{
171175
Image: imgCore,
172-
Reference: d.Ref,
176+
Reference: d.ref,
173177
}, nil
174178
}
175179

@@ -196,6 +200,7 @@ func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) {
196200
func (d *Descriptor) remoteImage() *remoteImage {
197201
return &remoteImage{
198202
fetcher: d.fetcher,
203+
ref: d.ref,
199204
manifest: d.Manifest,
200205
mediaType: d.MediaType,
201206
descriptor: &d.Descriptor,
@@ -205,38 +210,70 @@ func (d *Descriptor) remoteImage() *remoteImage {
205210
func (d *Descriptor) remoteIndex() *remoteIndex {
206211
return &remoteIndex{
207212
fetcher: d.fetcher,
213+
ref: d.ref,
208214
manifest: d.Manifest,
209215
mediaType: d.MediaType,
210216
descriptor: &d.Descriptor,
211217
}
212218
}
213219

220+
type resource interface {
221+
Scheme() string
222+
RegistryStr() string
223+
Scope(string) string
224+
225+
authn.Resource
226+
}
227+
214228
// fetcher implements methods for reading from a registry.
215229
type fetcher struct {
216-
Ref name.Reference
217-
Client *http.Client
230+
target resource
231+
client *http.Client
218232
context context.Context
219233
}
220234

221-
func makeFetcher(ref name.Reference, o *options) (*fetcher, error) {
222-
tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, []string{ref.Scope(transport.PullScope)})
235+
func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, error) {
236+
auth := o.auth
237+
if o.keychain != nil {
238+
kauth, err := o.keychain.Resolve(target)
239+
if err != nil {
240+
return nil, err
241+
}
242+
auth = kauth
243+
}
244+
245+
reg, ok := target.(name.Registry)
246+
if !ok {
247+
repo, ok := target.(name.Repository)
248+
if !ok {
249+
return nil, fmt.Errorf("unexpected resource: %T", target)
250+
}
251+
reg = repo.Registry
252+
}
253+
254+
tr, err := transport.NewWithContext(ctx, reg, auth, o.transport, []string{target.Scope(transport.PullScope)})
223255
if err != nil {
224256
return nil, err
225257
}
226258
return &fetcher{
227-
Ref: ref,
228-
Client: &http.Client{Transport: tr},
229-
context: o.context,
259+
target: target,
260+
client: &http.Client{Transport: tr},
261+
context: ctx,
230262
}, nil
231263
}
232264

233265
// url returns a url.Url for the specified path in the context of this remote image reference.
234266
func (f *fetcher) url(resource, identifier string) url.URL {
235-
return url.URL{
236-
Scheme: f.Ref.Context().Registry.Scheme(),
237-
Host: f.Ref.Context().RegistryStr(),
238-
Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier),
267+
u := url.URL{
268+
Scheme: f.target.Scheme(),
269+
Host: f.target.RegistryStr(),
270+
// Default path if this is not a repository.
271+
Path: "/v2/_catalog",
272+
}
273+
if repo, ok := f.target.(name.Repository); ok {
274+
u.Path = fmt.Sprintf("/v2/%s/%s/%s", repo.RepositoryStr(), resource, identifier)
239275
}
276+
return u
240277
}
241278

242279
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#referrers-tag-schema
@@ -253,7 +290,7 @@ func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string,
253290
}
254291
req.Header.Set("Accept", string(types.OCIImageIndex))
255292

256-
resp, err := f.Client.Do(req)
293+
resp, err := f.client.Do(req)
257294
if err != nil {
258295
return nil, err
259296
}
@@ -271,7 +308,7 @@ func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string,
271308
}
272309

273310
// The registry doesn't support the Referrers API endpoint, so we'll use the fallback tag scheme.
274-
b, _, err := f.fetchManifest(fallbackTag(d), []types.MediaType{types.OCIImageIndex})
311+
b, _, err := f.fetchManifest(ctx, fallbackTag(d), []types.MediaType{types.OCIImageIndex})
275312
if err != nil {
276313
return nil, err
277314
}
@@ -289,7 +326,7 @@ func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string,
289326
return filterReferrersResponse(filter, &im), nil
290327
}
291328

292-
func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) {
329+
func (f *fetcher) fetchManifest(ctx context.Context, ref name.Reference, acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) {
293330
u := f.url("manifests", ref.Identifier())
294331
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
295332
if err != nil {
@@ -301,7 +338,7 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType
301338
}
302339
req.Header.Set("Accept", strings.Join(accept, ","))
303340

304-
resp, err := f.Client.Do(req.WithContext(f.context))
341+
resp, err := f.client.Do(req.WithContext(ctx))
305342
if err != nil {
306343
return nil, nil, err
307344
}
@@ -332,7 +369,7 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType
332369
// Validate the digest matches what we asked for, if pulling by digest.
333370
if dgst, ok := ref.(name.Digest); ok {
334371
if digest.String() != dgst.DigestStr() {
335-
return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref)
372+
return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), ref)
336373
}
337374
}
338375

@@ -363,7 +400,7 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType
363400
return manifest, &desc, nil
364401
}
365402

366-
func (f *fetcher) headManifest(ref name.Reference, acceptable []types.MediaType) (*v1.Descriptor, error) {
403+
func (f *fetcher) headManifest(ctx context.Context, ref name.Reference, acceptable []types.MediaType) (*v1.Descriptor, error) {
367404
u := f.url("manifests", ref.Identifier())
368405
req, err := http.NewRequest(http.MethodHead, u.String(), nil)
369406
if err != nil {
@@ -375,7 +412,7 @@ func (f *fetcher) headManifest(ref name.Reference, acceptable []types.MediaType)
375412
}
376413
req.Header.Set("Accept", strings.Join(accept, ","))
377414

378-
resp, err := f.Client.Do(req.WithContext(f.context))
415+
resp, err := f.client.Do(req.WithContext(ctx))
379416
if err != nil {
380417
return nil, err
381418
}
@@ -408,7 +445,7 @@ func (f *fetcher) headManifest(ref name.Reference, acceptable []types.MediaType)
408445
// Validate the digest matches what we asked for, if pulling by digest.
409446
if dgst, ok := ref.(name.Digest); ok {
410447
if digest.String() != dgst.DigestStr() {
411-
return nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref)
448+
return nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), ref)
412449
}
413450
}
414451

@@ -427,7 +464,7 @@ func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.Read
427464
return nil, err
428465
}
429466

430-
resp, err := f.Client.Do(req.WithContext(ctx))
467+
resp, err := f.client.Do(req.WithContext(ctx))
431468
if err != nil {
432469
return nil, redact.Error(err)
433470
}
@@ -458,7 +495,7 @@ func (f *fetcher) headBlob(h v1.Hash) (*http.Response, error) {
458495
return nil, err
459496
}
460497

461-
resp, err := f.Client.Do(req.WithContext(f.context))
498+
resp, err := f.client.Do(req.WithContext(f.context))
462499
if err != nil {
463500
return nil, redact.Error(err)
464501
}
@@ -478,7 +515,7 @@ func (f *fetcher) blobExists(h v1.Hash) (bool, error) {
478515
return false, err
479516
}
480517

481-
resp, err := f.Client.Do(req.WithContext(f.context))
518+
resp, err := f.client.Do(req.WithContext(f.context))
482519
if err != nil {
483520
return false, redact.Error(err)
484521
}

‎pkg/v1/remote/descriptor_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,8 +225,8 @@ func TestHead_MissingHeaders(t *testing.T) {
225225
func TestRedactFetchBlob(t *testing.T) {
226226
ctx := context.Background()
227227
f := fetcher{
228-
Ref: mustNewTag(t, "original.com/repo:latest"),
229-
Client: &http.Client{
228+
target: mustNewTag(t, "original.com/repo:latest").Context(),
229+
client: &http.Client{
230230
Transport: errTransport{},
231231
},
232232
context: ctx,

‎pkg/v1/remote/image.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ var acceptableImageMediaTypes = []types.MediaType{
3838
// remoteImage accesses an image from a remote registry
3939
type remoteImage struct {
4040
fetcher
41+
ref name.Reference
4142
manifestLock sync.Mutex // Protects manifest
4243
manifest []byte
4344
configLock sync.Mutex // Protects config
@@ -84,7 +85,7 @@ func (r *remoteImage) RawManifest() ([]byte, error) {
8485
// NOTE(jonjohnsonjr): We should never get here because the public entrypoints
8586
// do type-checking via remote.Descriptor. I've left this here for tests that
8687
// directly instantiate a remoteImage.
87-
manifest, desc, err := r.fetchManifest(r.Ref, acceptableImageMediaTypes)
88+
manifest, desc, err := r.fetchManifest(r.context, r.ref, acceptableImageMediaTypes)
8889
if err != nil {
8990
return nil, err
9091
}
@@ -186,7 +187,7 @@ func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) {
186187
return nil, err
187188
}
188189

189-
resp, err := rl.ri.Client.Do(req.WithContext(ctx))
190+
resp, err := rl.ri.client.Do(req.WithContext(ctx))
190191
if err != nil {
191192
lastErr = err
192193
continue

0 commit comments

Comments
 (0)