Skip to content

Commit afd15f1

Browse files
jonjohnsonjrjonjohnsonjr
authored
Add --all-tags flag to crane cp (#1682)
This will copy every tag in the src repo to dst. This adds a --no-clobber flag that crane cp will respect that avoids overwriting an existing tag (at the time of command invocation).
1 parent 5438948 commit afd15f1

File tree

4 files changed

+165
-5
lines changed

4 files changed

+165
-5
lines changed

‎cmd/crane/cmd/copy.go‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,36 @@
1515
package cmd
1616

1717
import (
18+
"runtime"
19+
1820
"github.com/google/go-containerregistry/pkg/crane"
1921
"github.com/spf13/cobra"
2022
)
2123

2224
// NewCmdCopy creates a new cobra.Command for the copy subcommand.
2325
func NewCmdCopy(options *[]crane.Option) *cobra.Command {
24-
return &cobra.Command{
26+
allTags := false
27+
noclobber := false
28+
jobs := runtime.GOMAXPROCS(0)
29+
cmd := &cobra.Command{
2530
Use: "copy SRC DST",
2631
Aliases: []string{"cp"},
2732
Short: "Efficiently copy a remote image from src to dst while retaining the digest value",
2833
Args: cobra.ExactArgs(2),
2934
RunE: func(_ *cobra.Command, args []string) error {
35+
opts := append(*options, crane.WithJobs(jobs), crane.WithNoClobber(noclobber))
3036
src, dst := args[0], args[1]
31-
return crane.Copy(src, dst, *options...)
37+
if allTags {
38+
return crane.CopyRepository(src, dst, opts...)
39+
}
40+
41+
return crane.Copy(src, dst, opts...)
3242
},
3343
}
44+
45+
cmd.Flags().BoolVarP(&allTags, "all-tags", "a", false, "(Optional) if true, copy all tags from SRC to DST")
46+
cmd.Flags().BoolVarP(&noclobber, "no-clobber", "n", false, "(Optional) if true, avoid overwriting existing tags in DST")
47+
cmd.Flags().IntVarP(&jobs, "jobs", "j", 0, "(Optional) The maximum number of concurrent copies, defaults to GOMAXPROCS")
48+
49+
return cmd
3450
}

‎cmd/crane/doc/crane_copy.md‎

Lines changed: 4 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/crane/copy.go‎

Lines changed: 119 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,15 @@
1515
package crane
1616

1717
import (
18+
"errors"
1819
"fmt"
20+
"net/http"
1921

2022
"github.com/google/go-containerregistry/pkg/logs"
2123
"github.com/google/go-containerregistry/pkg/name"
2224
"github.com/google/go-containerregistry/pkg/v1/remote"
25+
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
26+
"golang.org/x/sync/errgroup"
2327
)
2428

2529
// Copy copies a remote image or index from src to dst.
@@ -35,12 +39,31 @@ func Copy(src, dst string, opt ...Option) error {
3539
return fmt.Errorf("parsing reference for %q: %w", dst, err)
3640
}
3741

38-
pusher, err := remote.NewPusher(o.Remote...)
42+
puller, err := remote.NewPuller(o.Remote...)
3943
if err != nil {
4044
return err
4145
}
4246

43-
puller, err := remote.NewPuller(o.Remote...)
47+
if tag, ok := dstRef.(name.Tag); ok {
48+
if o.noclobber {
49+
logs.Progress.Printf("Checking existing tag %v", tag)
50+
head, err := puller.Head(o.ctx, tag)
51+
var terr *transport.Error
52+
if errors.As(err, &terr) {
53+
if terr.StatusCode != http.StatusNotFound && terr.StatusCode != http.StatusForbidden {
54+
return err
55+
}
56+
} else if err != nil {
57+
return err
58+
}
59+
60+
if head != nil {
61+
return fmt.Errorf("refusing to clobber existing tag %s@%s", tag, head.Digest)
62+
}
63+
}
64+
}
65+
66+
pusher, err := remote.NewPusher(o.Remote...)
4467
if err != nil {
4568
return err
4669
}
@@ -62,3 +85,97 @@ func Copy(src, dst string, opt ...Option) error {
6285
}
6386
return pusher.Push(o.ctx, dstRef, img)
6487
}
88+
89+
// CopyRepository copies every tag from src to dst.
90+
func CopyRepository(src, dst string, opt ...Option) error {
91+
o := makeOptions(opt...)
92+
93+
srcRepo, err := name.NewRepository(src, o.Name...)
94+
if err != nil {
95+
return err
96+
}
97+
98+
dstRepo, err := name.NewRepository(dst, o.Name...)
99+
if err != nil {
100+
return fmt.Errorf("parsing reference for %q: %w", dst, err)
101+
}
102+
103+
puller, err := remote.NewPuller(o.Remote...)
104+
if err != nil {
105+
return err
106+
}
107+
108+
ignoredTags := map[string]struct{}{}
109+
if o.noclobber {
110+
// TODO: It would be good to propagate noclobber down into remote so we can use Etags.
111+
have, err := puller.List(o.ctx, dstRepo)
112+
if err != nil {
113+
var terr *transport.Error
114+
if errors.As(err, &terr) {
115+
// Some registries create repository on first push, so listing tags will fail.
116+
// If we see 404 or 403, assume we failed because the repository hasn't been created yet.
117+
if !(terr.StatusCode == http.StatusNotFound || terr.StatusCode == http.StatusForbidden) {
118+
return err
119+
}
120+
} else {
121+
return err
122+
}
123+
}
124+
for _, tag := range have {
125+
ignoredTags[tag] = struct{}{}
126+
}
127+
}
128+
129+
pusher, err := remote.NewPusher(o.Remote...)
130+
if err != nil {
131+
return err
132+
}
133+
134+
lister, err := puller.Lister(o.ctx, srcRepo)
135+
if err != nil {
136+
return err
137+
}
138+
139+
g, ctx := errgroup.WithContext(o.ctx)
140+
g.SetLimit(o.jobs)
141+
142+
for lister.HasNext() {
143+
tags, err := lister.Next(ctx)
144+
if err != nil {
145+
return err
146+
}
147+
148+
for _, tag := range tags.Tags {
149+
tag := tag
150+
151+
if o.noclobber {
152+
if _, ok := ignoredTags[tag]; ok {
153+
logs.Progress.Printf("Skipping %s due to no-clobber", tag)
154+
continue
155+
}
156+
}
157+
158+
g.Go(func() error {
159+
srcTag, err := name.ParseReference(src+":"+tag, o.Name...)
160+
if err != nil {
161+
return fmt.Errorf("failed to parse tag: %w", err)
162+
}
163+
dstTag, err := name.ParseReference(dst+":"+tag, o.Name...)
164+
if err != nil {
165+
return fmt.Errorf("failed to parse tag: %w", err)
166+
}
167+
168+
logs.Progress.Printf("Fetching %s", srcTag)
169+
desc, err := puller.Get(ctx, srcTag)
170+
if err != nil {
171+
return err
172+
}
173+
174+
logs.Progress.Printf("Pushing %s", dstTag)
175+
return pusher.Push(ctx, dstTag, desc)
176+
})
177+
}
178+
}
179+
180+
return g.Wait()
181+
}

‎pkg/crane/options.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,11 @@ type Options struct {
3232
Platform *v1.Platform
3333
Keychain authn.Keychain
3434

35+
auth authn.Authenticator
3536
transport http.RoundTripper
3637
insecure bool
38+
jobs int
39+
noclobber bool
3740
ctx context.Context
3841
}
3942

@@ -51,6 +54,7 @@ func makeOptions(opts ...Option) Options {
5154
remote.WithAuthFromKeychain(authn.DefaultKeychain),
5255
},
5356
Keychain: authn.DefaultKeychain,
57+
jobs: 4,
5458
ctx: context.Background(),
5559
}
5660

@@ -124,6 +128,7 @@ func WithAuth(auth authn.Authenticator) Option {
124128
return func(o *Options) {
125129
// Replace the default keychain at position 0.
126130
o.Remote[0] = remote.WithAuth(auth)
131+
o.auth = auth
127132
}
128133
}
129134

@@ -150,3 +155,22 @@ func WithContext(ctx context.Context) Option {
150155
o.Remote = append(o.Remote, remote.WithContext(ctx))
151156
}
152157
}
158+
159+
// WithJobs sets the number of concurrent jobs to run.
160+
//
161+
// The default number of jobs is GOMAXPROCS.
162+
func WithJobs(jobs int) Option {
163+
return func(o *Options) {
164+
if jobs > 0 {
165+
o.jobs = jobs
166+
}
167+
o.Remote = append(o.Remote, remote.WithJobs(o.jobs))
168+
}
169+
}
170+
171+
// WithNoClobber modifies behavior to avoid overwriting existing tags, if possible.
172+
func WithNoClobber(noclobber bool) Option {
173+
return func(o *Options) {
174+
o.noclobber = noclobber
175+
}
176+
}

0 commit comments

Comments
 (0)