Skip to content

Commit bf02f0a

Browse files
stefanprodanstefanprodan
authored
Merge pull request #1026 from fluxcd/storage-refactoring
artifact: Refactor storage package structure
2 parents ee36d78 + e2dd3d1 commit bf02f0a

11 files changed

Lines changed: 1943 additions & 1598 deletions

File tree

‎README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
- **[github.com/fluxcd/pkg/ssh](./ssh)** - SSH host key scanning and management
1818

1919
### Artifact Storage & Serving
20-
2120
- **[github.com/fluxcd/pkg/artifact](./artifact)** - Artifact Management SDK
2221
- **[github.com/fluxcd/pkg/artifact/config](./artifact/config)** - Configuration management of artifact storage and serving
2322
- **[github.com/fluxcd/pkg/artifact/digest](./artifact/digest)** - Multi-algorithm digest computation (SHA1, SHA256, SHA512, BLAKE3)

‎artifact/storage/archive.go‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
/*
2+
Copyright 2025 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package storage
18+
19+
import (
20+
"archive/tar"
21+
"compress/gzip"
22+
"fmt"
23+
"io"
24+
"os"
25+
"path/filepath"
26+
"strings"
27+
"time"
28+
29+
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
30+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31+
32+
"github.com/fluxcd/pkg/apis/meta"
33+
"github.com/fluxcd/pkg/oci"
34+
"github.com/fluxcd/pkg/sourceignore"
35+
36+
intdigest "github.com/fluxcd/pkg/artifact/digest"
37+
)
38+
39+
const (
40+
// DefaultFileMode is the permission mode applied to files inside an artifact archive.
41+
DefaultFileMode int64 = 0o600
42+
// DefaultDirMode is the permission mode applied to all directories inside an artifact archive.
43+
DefaultDirMode int64 = 0o750
44+
// DefaultExeFileMode is the permission mode applied to executable files inside an artifact archive.
45+
DefaultExeFileMode int64 = 0o700
46+
)
47+
48+
// writeCounter is an implementation of io.Writer
49+
// that only records the number of bytes written.
50+
type writeCounter struct {
51+
written int64
52+
}
53+
54+
// Write implements the io.Writer interface.
55+
func (wc *writeCounter) Write(p []byte) (int, error) {
56+
n := len(p)
57+
wc.written += int64(n)
58+
return n, nil
59+
}
60+
61+
// ArchiveFileFilter must return true if a file should not be included
62+
// in the archive after inspecting the given path and/or os.FileInfo.
63+
type ArchiveFileFilter func(p string, fi os.FileInfo) bool
64+
65+
// SourceIgnoreFilter returns an ArchiveFileFilter that filters out files matching
66+
// sourceignore.VCSPatterns and any of the provided patterns.
67+
// If an empty gitignore.Pattern slice is given, the matcher is set to sourceignore.NewDefaultMatcher.
68+
func SourceIgnoreFilter(ps []gitignore.Pattern, domain []string) ArchiveFileFilter {
69+
matcher := sourceignore.NewDefaultMatcher(ps, domain)
70+
if len(ps) > 0 {
71+
ps = append(sourceignore.VCSPatterns(domain), ps...)
72+
matcher = sourceignore.NewMatcher(ps)
73+
}
74+
return func(p string, fi os.FileInfo) bool {
75+
return matcher.Match(strings.Split(p, string(filepath.Separator)), fi.IsDir())
76+
}
77+
}
78+
79+
// Archive atomically archives the given directory as a tarball to the given meta.Artifact path,
80+
// excluding directories and any ArchiveFileFilter matches. While archiving, any environment
81+
// specific data (for example, the user and group name) is stripped from file headers.
82+
// If successful, it sets the digest and last update time on the artifact.
83+
func (s Storage) Archive(artifact *meta.Artifact, dir string, filter ArchiveFileFilter) (err error) {
84+
if f, err := os.Stat(dir); os.IsNotExist(err) || !f.IsDir() {
85+
return fmt.Errorf("invalid dir path: %s", dir)
86+
}
87+
88+
localPath := s.LocalPath(*artifact)
89+
tf, err := os.CreateTemp(filepath.Split(localPath))
90+
if err != nil {
91+
return err
92+
}
93+
tmpName := tf.Name()
94+
defer func() {
95+
if err != nil {
96+
os.Remove(tmpName)
97+
}
98+
}()
99+
100+
d := intdigest.Canonical.Digester()
101+
sz := &writeCounter{}
102+
mw := io.MultiWriter(d.Hash(), tf, sz)
103+
104+
gw := gzip.NewWriter(mw)
105+
tw := tar.NewWriter(gw)
106+
if err := filepath.Walk(dir, func(p string, fi os.FileInfo, err error) error {
107+
if err != nil {
108+
return err
109+
}
110+
111+
// Ignore anything that is not a file or directories e.g. symlinks
112+
if m := fi.Mode(); !(m.IsRegular() || m.IsDir()) {
113+
return nil
114+
}
115+
116+
// Skip filtered files
117+
if filter != nil && filter(p, fi) {
118+
return nil
119+
}
120+
121+
header, err := tar.FileInfoHeader(fi, p)
122+
if err != nil {
123+
return err
124+
}
125+
126+
// The name needs to be modified to maintain directory structure
127+
// as tar.FileInfoHeader only has access to the base name of the file.
128+
// Ref: https://golang.org/src/archive/tar/common.go?#L626
129+
relFilePath := p
130+
if filepath.IsAbs(dir) {
131+
relFilePath, err = filepath.Rel(dir, p)
132+
if err != nil {
133+
return err
134+
}
135+
}
136+
sanitizeHeader(relFilePath, header)
137+
138+
if err := tw.WriteHeader(header); err != nil {
139+
return err
140+
}
141+
142+
if !fi.Mode().IsRegular() {
143+
return nil
144+
}
145+
f, err := os.Open(p)
146+
if err != nil {
147+
f.Close()
148+
return err
149+
}
150+
if _, err := io.Copy(tw, f); err != nil {
151+
f.Close()
152+
return err
153+
}
154+
return f.Close()
155+
}); err != nil {
156+
tw.Close()
157+
gw.Close()
158+
tf.Close()
159+
return err
160+
}
161+
162+
if err := tw.Close(); err != nil {
163+
gw.Close()
164+
tf.Close()
165+
return err
166+
}
167+
if err := gw.Close(); err != nil {
168+
tf.Close()
169+
return err
170+
}
171+
if err := tf.Close(); err != nil {
172+
return err
173+
}
174+
175+
if err := os.Chmod(tmpName, 0o600); err != nil {
176+
return err
177+
}
178+
179+
if err := oci.RenameWithFallback(tmpName, localPath); err != nil {
180+
return err
181+
}
182+
183+
artifact.Digest = d.Digest().String()
184+
artifact.LastUpdateTime = metav1.Now()
185+
artifact.Size = &sz.written
186+
187+
return nil
188+
}
189+
190+
// sanitizeHeader modifies the tar.Header to be relative to the root of the
191+
// archive and removes any environment specific data.
192+
func sanitizeHeader(relP string, h *tar.Header) {
193+
// Modify the name to be relative to the root of the archive,
194+
// this ensures we maintain the same structure when extracting.
195+
h.Name = relP
196+
197+
// We want to remove any environment specific data as well, this
198+
// ensures the checksum is purely content based.
199+
h.Gid = 0
200+
h.Uid = 0
201+
h.Uname = ""
202+
h.Gname = ""
203+
h.ModTime = time.Time{}
204+
h.AccessTime = time.Time{}
205+
h.ChangeTime = time.Time{}
206+
207+
// Override the mode to be the default for the type of file.
208+
setDefaultMode(h)
209+
}
210+
211+
// setDefaultMode sets the default mode for the given header.
212+
func setDefaultMode(h *tar.Header) {
213+
if h.FileInfo().IsDir() {
214+
h.Mode = DefaultDirMode
215+
return
216+
}
217+
218+
if h.FileInfo().Mode().IsRegular() {
219+
mode := h.FileInfo().Mode()
220+
if mode&os.ModeType == 0 && mode&0o111 != 0 {
221+
h.Mode = DefaultExeFileMode
222+
return
223+
}
224+
h.Mode = DefaultFileMode
225+
return
226+
}
227+
}

0 commit comments

Comments
 (0)