Skip to content

Commit 07c767c

Browse files
jonjohnsonjrjonjohnsonjr
authored
Add remote.Puller (#1644)
This PR adds a Puller implementation and uses it in `crane ls` and `crane catalog` to stream results for each page.
1 parent 9f68710 commit 07c767c

File tree

7 files changed

+382
-159
lines changed

7 files changed

+382
-159
lines changed

‎cmd/crane/cmd/catalog.go‎

Lines changed: 45 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -15,40 +15,62 @@
1515
package cmd
1616

1717
import (
18+
"context"
1819
"fmt"
19-
"os"
20-
"strings"
20+
"path"
2121

2222
"github.com/google/go-containerregistry/pkg/crane"
23+
"github.com/google/go-containerregistry/pkg/name"
24+
"github.com/google/go-containerregistry/pkg/v1/remote"
2325
"github.com/spf13/cobra"
2426
)
2527

26-
// NewCmdCatalog creates a new cobra.Command for the repos subcommand.
28+
// NewCmdCatalog creates a new cobra.Command for the catalog subcommand.
2729
func NewCmdCatalog(options *[]crane.Option, argv ...string) *cobra.Command {
28-
if len(argv) == 0 {
29-
argv = []string{os.Args[0]}
30+
var fullRef bool
31+
cmd := &cobra.Command{
32+
Use: "catalog REGISTRY",
33+
Short: "List the repos in a registry",
34+
Args: cobra.ExactArgs(1),
35+
RunE: func(cmd *cobra.Command, args []string) error {
36+
o := crane.GetOptions(*options...)
37+
38+
return catalog(cmd.Context(), args[0], fullRef, o)
39+
},
3040
}
41+
cmd.Flags().BoolVar(&fullRef, "full-ref", false, "(Optional) if true, print the full image reference")
3142

32-
baseCmd := strings.Join(argv, " ")
33-
eg := fmt.Sprintf(` # list the repos for reg.example.com
34-
$ %s catalog reg.example.com`, baseCmd)
35-
36-
return &cobra.Command{
37-
Use: "catalog [REGISTRY]",
38-
Short: "List the repos in a registry",
39-
Example: eg,
40-
Args: cobra.ExactArgs(1),
41-
RunE: func(_ *cobra.Command, args []string) error {
42-
reg := args[0]
43-
repos, err := crane.Catalog(reg, *options...)
44-
if err != nil {
45-
return fmt.Errorf("reading repos for %s: %w", reg, err)
46-
}
43+
return cmd
44+
}
45+
46+
func catalog(ctx context.Context, src string, fullRef bool, o crane.Options) error {
47+
reg, err := name.NewRegistry(src, o.Name...)
48+
if err != nil {
49+
return fmt.Errorf("parsing reg %q: %w", src, err)
50+
}
4751

48-
for _, repo := range repos {
52+
puller, err := remote.NewPuller(o.Remote...)
53+
if err != nil {
54+
return err
55+
}
56+
57+
catalogger, err := puller.Catalogger(ctx, reg)
58+
if err != nil {
59+
return fmt.Errorf("reading tags for %s: %w", reg, err)
60+
}
61+
62+
for catalogger.HasNext() {
63+
repos, err := catalogger.Next(ctx)
64+
if err != nil {
65+
return err
66+
}
67+
for _, repo := range repos.Repos {
68+
if fullRef {
69+
fmt.Println(path.Join(src, repo))
70+
} else {
4971
fmt.Println(repo)
5072
}
51-
return nil
52-
},
73+
}
5374
}
75+
return nil
5476
}

‎cmd/crane/cmd/list.go‎

Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@
1515
package cmd
1616

1717
import (
18+
"context"
1819
"fmt"
1920
"strings"
2021

2122
"github.com/google/go-containerregistry/pkg/crane"
2223
"github.com/google/go-containerregistry/pkg/name"
24+
"github.com/google/go-containerregistry/pkg/v1/remote"
2325
"github.com/spf13/cobra"
2426
)
2527

@@ -30,33 +32,49 @@ func NewCmdList(options *[]crane.Option) *cobra.Command {
3032
Use: "ls REPO",
3133
Short: "List the tags in a repo",
3234
Args: cobra.ExactArgs(1),
33-
RunE: func(_ *cobra.Command, args []string) error {
34-
repo := args[0]
35-
tags, err := crane.ListTags(repo, *options...)
36-
if err != nil {
37-
return fmt.Errorf("reading tags for %s: %w", repo, err)
38-
}
39-
40-
r, err := name.NewRepository(repo)
41-
if err != nil {
42-
return err
43-
}
35+
RunE: func(cmd *cobra.Command, args []string) error {
36+
o := crane.GetOptions(*options...)
4437

45-
for _, tag := range tags {
46-
if omitDigestTags && strings.HasPrefix(tag, "sha256-") {
47-
continue
48-
}
49-
50-
if fullRef {
51-
fmt.Println(r.Tag(tag))
52-
} else {
53-
fmt.Println(tag)
54-
}
55-
}
56-
return nil
38+
return list(cmd.Context(), args[0], fullRef, omitDigestTags, o)
5739
},
5840
}
5941
cmd.Flags().BoolVar(&fullRef, "full-ref", false, "(Optional) if true, print the full image reference")
6042
cmd.Flags().BoolVar(&omitDigestTags, "omit-digest-tags", false, "(Optional), if true, omit digest tags (e.g., ':sha256-...')")
6143
return cmd
6244
}
45+
46+
func list(ctx context.Context, src string, fullRef, omitDigestTags bool, o crane.Options) error {
47+
repo, err := name.NewRepository(src, o.Name...)
48+
if err != nil {
49+
return fmt.Errorf("parsing repo %q: %w", src, err)
50+
}
51+
52+
puller, err := remote.NewPuller(o.Remote...)
53+
if err != nil {
54+
return err
55+
}
56+
57+
lister, err := puller.Lister(ctx, repo)
58+
if err != nil {
59+
return fmt.Errorf("reading tags for %s: %w", repo, err)
60+
}
61+
62+
for lister.HasNext() {
63+
tags, err := lister.Next(ctx)
64+
if err != nil {
65+
return err
66+
}
67+
for _, tag := range tags.Tags {
68+
if omitDigestTags && strings.HasPrefix(tag, "sha256-") {
69+
continue
70+
}
71+
72+
if fullRef {
73+
fmt.Println(repo.Tag(tag))
74+
} else {
75+
fmt.Println(tag)
76+
}
77+
}
78+
}
79+
return nil
80+
}

‎cmd/crane/doc/crane_catalog.md‎

Lines changed: 3 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/v1/remote/catalog.go‎

Lines changed: 64 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@ import (
2525
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
2626
)
2727

28-
type catalog struct {
28+
type Catalogs struct {
2929
Repos []string `json:"repositories"`
30+
Next string `json:"next,omitempty"`
3031
}
3132

3233
// CatalogPage calls /_catalog, returning the list of repositories on the registry.
@@ -61,7 +62,7 @@ func CatalogPage(target name.Registry, last string, n int, options ...Option) ([
6162
return nil, err
6263
}
6364

64-
var parsed catalog
65+
var parsed Catalogs
6566
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
6667
return nil, err
6768
}
@@ -75,70 +76,82 @@ func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]st
7576
if err != nil {
7677
return nil, err
7778
}
78-
f, err := makeFetcher(o.context, target, o)
79+
80+
// WithContext overrides the ctx passed directly.
81+
if o.context != context.Background() {
82+
ctx = o.context
83+
}
84+
85+
return newPuller(o).Catalog(ctx, target)
86+
}
87+
88+
func (f *fetcher) catalogPage(ctx context.Context, reg name.Registry, next string) (*Catalogs, error) {
89+
if next == "" {
90+
uri := &url.URL{
91+
Scheme: reg.Scheme(),
92+
Host: reg.RegistryStr(),
93+
Path: "/v2/_catalog",
94+
}
95+
if f.pageSize > 0 {
96+
uri.RawQuery = fmt.Sprintf("n=%d", f.pageSize)
97+
}
98+
next = uri.String()
99+
}
100+
101+
req, err := http.NewRequestWithContext(ctx, "GET", next, nil)
79102
if err != nil {
80103
return nil, err
81104
}
82105

83-
uri := &url.URL{
84-
Scheme: target.Scheme(),
85-
Host: target.RegistryStr(),
86-
Path: "/v2/_catalog",
106+
resp, err := f.client.Do(req)
107+
if err != nil {
108+
return nil, err
87109
}
88-
if o.pageSize > 0 {
89-
uri.RawQuery = fmt.Sprintf("n=%d", o.pageSize)
110+
111+
if err := transport.CheckError(resp, http.StatusOK); err != nil {
112+
return nil, err
90113
}
91114

92-
// WithContext overrides the ctx passed directly.
93-
if o.context != context.Background() {
94-
ctx = o.context
115+
parsed := Catalogs{}
116+
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
117+
return nil, err
95118
}
96119

97-
var (
98-
parsed catalog
99-
repoList []string
100-
)
120+
if err := resp.Body.Close(); err != nil {
121+
return nil, err
122+
}
101123

102-
// get responses until there is no next page
103-
for {
104-
select {
105-
case <-ctx.Done():
106-
return nil, ctx.Err()
107-
default:
108-
}
124+
uri, err := getNextPageURL(resp)
125+
if err != nil {
126+
return nil, err
127+
}
109128

110-
req, err := http.NewRequest("GET", uri.String(), nil)
111-
if err != nil {
112-
return nil, err
113-
}
114-
req = req.WithContext(ctx)
129+
if uri != nil {
130+
parsed.Next = uri.String()
131+
}
115132

116-
resp, err := f.client.Do(req)
117-
if err != nil {
118-
return nil, err
119-
}
133+
return &parsed, nil
134+
}
120135

121-
if err := transport.CheckError(resp, http.StatusOK); err != nil {
122-
return nil, err
123-
}
136+
type Catalogger struct {
137+
f *fetcher
138+
reg name.Registry
124139

125-
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
126-
return nil, err
127-
}
128-
if err := resp.Body.Close(); err != nil {
129-
return nil, err
130-
}
140+
page *Catalogs
141+
err error
131142

132-
repoList = append(repoList, parsed.Repos...)
143+
needMore bool
144+
}
133145

134-
uri, err = getNextPageURL(resp)
135-
if err != nil {
136-
return nil, err
137-
}
138-
// no next page
139-
if uri == nil {
140-
break
141-
}
146+
func (l *Catalogger) Next(ctx context.Context) (*Catalogs, error) {
147+
if l.needMore {
148+
l.page, l.err = l.f.catalogPage(ctx, l.reg, l.page.Next)
149+
} else {
150+
l.needMore = true
142151
}
143-
return repoList, nil
152+
return l.page, l.err
153+
}
154+
155+
func (l *Catalogger) HasNext() bool {
156+
return l.page != nil && (!l.needMore || l.page.Next != "")
144157
}

‎pkg/v1/remote/descriptor.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,7 @@ type fetcher struct {
250250
client *http.Client
251251
context context.Context
252252
platform v1.Platform
253+
pageSize int
253254
}
254255

255256
func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, error) {
@@ -280,6 +281,7 @@ func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, er
280281
client: &http.Client{Transport: tr},
281282
context: ctx,
282283
platform: o.platform,
284+
pageSize: o.pageSize,
283285
}, nil
284286
}
285287

0 commit comments

Comments
 (0)