Skip to content

Commit 5c3d2d5

Browse files
tonistiigitonistiigi
committed
Implement incremental file sync using client session
Also exposes shared cache and garbage collection/prune for the source data. Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
1 parent 7cfcf74 commit 5c3d2d5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+2872
-185
lines changed

‎api/server/backend/build/backend.go‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@ import (
44
"fmt"
55

66
"github.com/docker/distribution/reference"
7+
"github.com/docker/docker/api/types"
78
"github.com/docker/docker/api/types/backend"
89
"github.com/docker/docker/builder"
9-
"github.com/docker/docker/builder/dockerfile"
10+
"github.com/docker/docker/builder/fscache"
1011
"github.com/docker/docker/image"
11-
"github.com/docker/docker/pkg/idtools"
1212
"github.com/docker/docker/pkg/stringid"
1313
"github.com/pkg/errors"
1414
"golang.org/x/net/context"
@@ -20,16 +20,21 @@ type ImageComponent interface {
2020
TagImageWithReference(image.ID, string, reference.Named) error
2121
}
2222

23+
// Builder defines interface for running a build
24+
type Builder interface {
25+
Build(context.Context, backend.BuildConfig) (*builder.Result, error)
26+
}
27+
2328
// Backend provides build functionality to the API router
2429
type Backend struct {
25-
manager *dockerfile.BuildManager
30+
builder Builder
31+
fsCache *fscache.FSCache
2632
imageComponent ImageComponent
2733
}
2834

2935
// NewBackend creates a new build backend from components
30-
func NewBackend(components ImageComponent, builderBackend builder.Backend, sg dockerfile.SessionGetter, idMappings *idtools.IDMappings) (*Backend, error) {
31-
manager := dockerfile.NewBuildManager(builderBackend, sg, idMappings)
32-
return &Backend{imageComponent: components, manager: manager}, nil
36+
func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache) (*Backend, error) {
37+
return &Backend{imageComponent: components, builder: builder, fsCache: fsCache}, nil
3338
}
3439

3540
// Build builds an image from a Source
@@ -40,7 +45,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
4045
return "", err
4146
}
4247

43-
build, err := b.manager.Build(ctx, config)
48+
build, err := b.builder.Build(ctx, config)
4449
if err != nil {
4550
return "", err
4651
}
@@ -58,6 +63,15 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
5863
return imageID, err
5964
}
6065

66+
// PruneCache removes all cached build sources
67+
func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
68+
size, err := b.fsCache.Prune()
69+
if err != nil {
70+
return nil, errors.Wrap(err, "failed to prune build cache")
71+
}
72+
return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
73+
}
74+
6175
func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {
6276
var fromID string
6377
if build.FromImage != nil {

‎api/server/router/build/backend.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package build
22

33
import (
4+
"github.com/docker/docker/api/types"
45
"github.com/docker/docker/api/types/backend"
56
"golang.org/x/net/context"
67
)
@@ -10,6 +11,9 @@ type Backend interface {
1011
// Build a Docker image returning the id of the image
1112
// TODO: make this return a reference instead of string
1213
Build(context.Context, backend.BuildConfig) (string, error)
14+
15+
// Prune build cache
16+
PruneCache(context.Context) (*types.BuildCachePruneReport, error)
1317
}
1418

1519
type experimentalProvider interface {

‎api/server/router/build/build.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,6 @@ func (r *buildRouter) Routes() []router.Route {
2424
func (r *buildRouter) initRoutes() {
2525
r.routes = []router.Route{
2626
router.NewPostRoute("/build", r.postBuild, router.WithCancel),
27+
router.NewPostRoute("/build/prune", r.postPrune, router.WithCancel),
2728
}
2829
}

‎api/server/router/build/build_routes.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,14 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
132132
return options, nil
133133
}
134134

135+
func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
136+
report, err := br.backend.PruneCache(ctx)
137+
if err != nil {
138+
return err
139+
}
140+
return httputils.WriteJSON(w, http.StatusOK, report)
141+
}
142+
135143
func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
136144
var (
137145
notVerboseBuffer = bytes.NewBuffer(nil)

‎api/server/router/system/system.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package system
22

33
import (
44
"github.com/docker/docker/api/server/router"
5+
"github.com/docker/docker/builder/fscache"
56
"github.com/docker/docker/daemon/cluster"
67
)
78

@@ -11,13 +12,15 @@ type systemRouter struct {
1112
backend Backend
1213
cluster *cluster.Cluster
1314
routes []router.Route
15+
builder *fscache.FSCache
1416
}
1517

1618
// NewRouter initializes a new system router
17-
func NewRouter(b Backend, c *cluster.Cluster) router.Router {
19+
func NewRouter(b Backend, c *cluster.Cluster, fscache *fscache.FSCache) router.Router {
1820
r := &systemRouter{
1921
backend: b,
2022
cluster: c,
23+
builder: fscache,
2124
}
2225

2326
r.routes = []router.Route{

‎api/server/router/system/system_routes.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
timetypes "github.com/docker/docker/api/types/time"
1818
"github.com/docker/docker/api/types/versions"
1919
"github.com/docker/docker/pkg/ioutils"
20+
pkgerrors "github.com/pkg/errors"
2021
"golang.org/x/net/context"
2122
)
2223

@@ -75,6 +76,11 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
7576
if err != nil {
7677
return err
7778
}
79+
builderSize, err := s.builder.DiskUsage()
80+
if err != nil {
81+
return pkgerrors.Wrap(err, "error getting build cache usage")
82+
}
83+
du.BuilderSize = builderSize
7884

7985
return httputils.WriteJSON(w, http.StatusOK, du)
8086
}

‎api/swagger.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4745,6 +4745,27 @@ paths:
47454745
schema:
47464746
$ref: "#/definitions/ErrorResponse"
47474747
tags: ["Image"]
4748+
/build/prune:
4749+
post:
4750+
summary: "Delete builder cache"
4751+
produces:
4752+
- "application/json"
4753+
operationId: "BuildPrune"
4754+
responses:
4755+
200:
4756+
description: "No error"
4757+
schema:
4758+
type: "object"
4759+
properties:
4760+
SpaceReclaimed:
4761+
description: "Disk space reclaimed in bytes"
4762+
type: "integer"
4763+
format: "int64"
4764+
500:
4765+
description: "Server error"
4766+
schema:
4767+
$ref: "#/definitions/ErrorResponse"
4768+
tags: ["Image"]
47484769
/images/create:
47494770
post:
47504771
summary: "Create an image"

‎api/types/types.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -489,10 +489,11 @@ type Runtime struct {
489489
// DiskUsage contains response of Engine API:
490490
// GET "/system/df"
491491
type DiskUsage struct {
492-
LayersSize int64
493-
Images []*ImageSummary
494-
Containers []*Container
495-
Volumes []*Volume
492+
LayersSize int64
493+
Images []*ImageSummary
494+
Containers []*Container
495+
Volumes []*Volume
496+
BuilderSize int64
496497
}
497498

498499
// ContainersPruneReport contains the response for Engine API:
@@ -516,6 +517,12 @@ type ImagesPruneReport struct {
516517
SpaceReclaimed uint64
517518
}
518519

520+
// BuildCachePruneReport contains the response for Engine API:
521+
// POST "/build/prune"
522+
type BuildCachePruneReport struct {
523+
SpaceReclaimed uint64
524+
}
525+
519526
// NetworksPruneReport contains the response for Engine API:
520527
// POST "/networks/prune"
521528
type NetworksPruneReport struct {

‎builder/dockerfile/builder.go‎

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"io/ioutil"
88
"runtime"
99
"strings"
10+
"time"
1011

1112
"github.com/Sirupsen/logrus"
1213
"github.com/docker/docker/api/types"
@@ -15,6 +16,7 @@ import (
1516
"github.com/docker/docker/builder"
1617
"github.com/docker/docker/builder/dockerfile/command"
1718
"github.com/docker/docker/builder/dockerfile/parser"
19+
"github.com/docker/docker/builder/fscache"
1820
"github.com/docker/docker/builder/remotecontext"
1921
"github.com/docker/docker/client/session"
2022
"github.com/docker/docker/pkg/archive"
@@ -52,16 +54,22 @@ type BuildManager struct {
5254
backend builder.Backend
5355
pathCache pathCache // TODO: make this persistent
5456
sg SessionGetter
57+
fsCache *fscache.FSCache
5558
}
5659

5760
// NewBuildManager creates a BuildManager
58-
func NewBuildManager(b builder.Backend, sg SessionGetter, idMappings *idtools.IDMappings) *BuildManager {
59-
return &BuildManager{
61+
func NewBuildManager(b builder.Backend, sg SessionGetter, fsCache *fscache.FSCache, idMappings *idtools.IDMappings) (*BuildManager, error) {
62+
bm := &BuildManager{
6063
backend: b,
6164
pathCache: &syncmap.Map{},
6265
sg: sg,
6366
archiver: chrootarchive.NewArchiver(idMappings),
67+
fsCache: fsCache,
6468
}
69+
if err := fsCache.RegisterTransport(remotecontext.ClientSessionRemote, NewClientSessionTransport()); err != nil {
70+
return nil, err
71+
}
72+
return bm, nil
6573
}
6674

6775
// Build starts a new build from a BuildConfig
@@ -75,13 +83,13 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
7583
if err != nil {
7684
return nil, err
7785
}
78-
if source != nil {
79-
defer func() {
86+
defer func() {
87+
if source != nil {
8088
if err := source.Close(); err != nil {
8189
logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
8290
}
83-
}()
84-
}
91+
}
92+
}()
8593

8694
// TODO @jhowardmsft LCOW support - this will require rework to allow both linux and Windows simultaneously.
8795
// This is an interim solution to hardcode to linux if LCOW is turned on.
@@ -95,8 +103,10 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
95103
ctx, cancel := context.WithCancel(ctx)
96104
defer cancel()
97105

98-
if err := bm.initializeClientSession(ctx, cancel, config.Options); err != nil {
106+
if src, err := bm.initializeClientSession(ctx, cancel, config.Options); err != nil {
99107
return nil, err
108+
} else if src != nil {
109+
source = src
100110
}
101111

102112
builderOptions := builderOptions{
@@ -111,20 +121,38 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
111121
return newBuilder(ctx, builderOptions).build(source, dockerfile)
112122
}
113123

114-
func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) error {
124+
func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) (builder.Source, error) {
115125
if options.SessionID == "" || bm.sg == nil {
116-
return nil
126+
return nil, nil
117127
}
118128
logrus.Debug("client is session enabled")
129+
130+
ctx, cancelCtx := context.WithTimeout(ctx, sessionConnectTimeout)
131+
defer cancelCtx()
132+
119133
c, err := bm.sg.Get(ctx, options.SessionID)
120134
if err != nil {
121-
return err
135+
return nil, err
122136
}
123137
go func() {
124138
<-c.Context().Done()
125139
cancel()
126140
}()
127-
return nil
141+
if options.RemoteContext == remotecontext.ClientSessionRemote {
142+
st := time.Now()
143+
csi, err := NewClientSessionSourceIdentifier(ctx, bm.sg,
144+
options.SessionID, []string{"/"})
145+
if err != nil {
146+
return nil, err
147+
}
148+
src, err := bm.fsCache.SyncFrom(ctx, csi)
149+
if err != nil {
150+
return nil, err
151+
}
152+
logrus.Debugf("sync-time: %v", time.Since(st))
153+
return src, nil
154+
}
155+
return nil, nil
128156
}
129157

130158
// builderOptions are the dependencies required by the builder

‎builder/dockerfile/clientsession.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package dockerfile
2+
3+
import (
4+
"time"
5+
6+
"github.com/docker/docker/builder/fscache"
7+
"github.com/docker/docker/builder/remotecontext"
8+
"github.com/docker/docker/client/session"
9+
"github.com/docker/docker/client/session/filesync"
10+
"github.com/pkg/errors"
11+
"golang.org/x/net/context"
12+
)
13+
14+
const sessionConnectTimeout = 5 * time.Second
15+
16+
// ClientSessionTransport is a transport for copying files from docker client
17+
// to the daemon.
18+
type ClientSessionTransport struct{}
19+
20+
// NewClientSessionTransport returns new ClientSessionTransport instance
21+
func NewClientSessionTransport() *ClientSessionTransport {
22+
return &ClientSessionTransport{}
23+
}
24+
25+
// Copy data from a remote to a destination directory.
26+
func (cst *ClientSessionTransport) Copy(ctx context.Context, id fscache.RemoteIdentifier, dest string, cu filesync.CacheUpdater) error {
27+
csi, ok := id.(*ClientSessionSourceIdentifier)
28+
if !ok {
29+
return errors.New("invalid identifier for client session")
30+
}
31+
32+
return filesync.FSSync(ctx, csi.caller, filesync.FSSendRequestOpt{
33+
SrcPaths: csi.srcPaths,
34+
DestDir: dest,
35+
CacheUpdater: cu,
36+
})
37+
}
38+
39+
// ClientSessionSourceIdentifier is an identifier that can be used for requesting
40+
// files from remote client
41+
type ClientSessionSourceIdentifier struct {
42+
srcPaths []string
43+
caller session.Caller
44+
sharedKey string
45+
uuid string
46+
}
47+
48+
// NewClientSessionSourceIdentifier returns new ClientSessionSourceIdentifier instance
49+
func NewClientSessionSourceIdentifier(ctx context.Context, sg SessionGetter, uuid string, sources []string) (*ClientSessionSourceIdentifier, error) {
50+
csi := &ClientSessionSourceIdentifier{
51+
uuid: uuid,
52+
srcPaths: sources,
53+
}
54+
caller, err := sg.Get(ctx, uuid)
55+
if err != nil {
56+
return nil, errors.Wrapf(err, "failed to get session for %s", uuid)
57+
}
58+
59+
csi.caller = caller
60+
return csi, nil
61+
}
62+
63+
// Transport returns transport identifier for remote identifier
64+
func (csi *ClientSessionSourceIdentifier) Transport() string {
65+
return remotecontext.ClientSessionRemote
66+
}
67+
68+
// SharedKey returns shared key for remote identifier. Shared key is used
69+
// for finding the base for a repeated transfer.
70+
func (csi *ClientSessionSourceIdentifier) SharedKey() string {
71+
return csi.caller.SharedKey()
72+
}
73+
74+
// Key returns unique key for remote identifier. Requests with same key return
75+
// same data.
76+
func (csi *ClientSessionSourceIdentifier) Key() string {
77+
return csi.uuid
78+
}

0 commit comments

Comments
 (0)