Implement incremental file sync using client session

Also exposes shared cache and garbage collection/prune
for the source data.

Signed-off-by: Tonis Tiigi <[email protected]>

api/server/backend/build/backend.go

@@ -4,11 +4,11 @@ import (
 	"fmt"
 
 	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
-	"github.com/docker/docker/builder/dockerfile"
+	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
@@ -20,16 +20,21 @@ type ImageComponent interface {
 	TagImageWithReference(image.ID, string, reference.Named) error
 }
 
+// Builder defines interface for running a build
+type Builder interface {
+	Build(context.Context, backend.BuildConfig) (*builder.Result, error)
+}
+
 // Backend provides build functionality to the API router
 type Backend struct {
-	manager        *dockerfile.BuildManager
+	builder        Builder
+	fsCache        *fscache.FSCache
 	imageComponent ImageComponent
 }
 
 // NewBackend creates a new build backend from components
-func NewBackend(components ImageComponent, builderBackend builder.Backend, sg dockerfile.SessionGetter, idMappings *idtools.IDMappings) (*Backend, error) {
-	manager := dockerfile.NewBuildManager(builderBackend, sg, idMappings)
-	return &Backend{imageComponent: components, manager: manager}, nil
+func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache) (*Backend, error) {
+	return &Backend{imageComponent: components, builder: builder, fsCache: fsCache}, nil
 }
 
 // Build builds an image from a Source
@@ -40,7 +45,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 		return "", err
 	}
 
-	build, err := b.manager.Build(ctx, config)
+	build, err := b.builder.Build(ctx, config)
 	if err != nil {
 		return "", err
 	}
@@ -58,6 +63,15 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 	return imageID, err
 }
 
+// PruneCache removes all cached build sources
+func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
+	size, err := b.fsCache.Prune()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to prune build cache")
+	}
+	return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
+}
+
 func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {
 	var fromID string
 	if build.FromImage != nil {

api/server/router/build/backend.go

@@ -1,6 +1,7 @@
 package build
 
 import (
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"golang.org/x/net/context"
 )
@@ -10,6 +11,9 @@ type Backend interface {
 	// Build a Docker image returning the id of the image
 	// TODO: make this return a reference instead of string
 	Build(context.Context, backend.BuildConfig) (string, error)
+
+	// Prune build cache
+	PruneCache(context.Context) (*types.BuildCachePruneReport, error)
 }
 
 type experimentalProvider interface {

api/server/router/build/build.go

@@ -24,5 +24,6 @@ func (r *buildRouter) Routes() []router.Route {
 func (r *buildRouter) initRoutes() {
 	r.routes = []router.Route{
 		router.NewPostRoute("/build", r.postBuild, router.WithCancel),
+		router.NewPostRoute("/build/prune", r.postPrune, router.WithCancel),
 	}
 }

api/server/router/build/build_routes.go

@@ -132,6 +132,14 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 	return options, nil
 }
 
+func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	report, err := br.backend.PruneCache(ctx)
+	if err != nil {
+		return err
+	}
+	return httputils.WriteJSON(w, http.StatusOK, report)
+}
+
 func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	var (
 		notVerboseBuffer = bytes.NewBuffer(nil)

api/server/router/system/system.go

@@ -2,6 +2,7 @@ package system
 
 import (
 	"github.com/docker/docker/api/server/router"
+	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/daemon/cluster"
 )
 
@@ -11,13 +12,15 @@ type systemRouter struct {
 	backend Backend
 	cluster *cluster.Cluster
 	routes  []router.Route
+	builder *fscache.FSCache
 }
 
 // NewRouter initializes a new system router
-func NewRouter(b Backend, c *cluster.Cluster) router.Router {
+func NewRouter(b Backend, c *cluster.Cluster, fscache *fscache.FSCache) router.Router {
 	r := &systemRouter{
 		backend: b,
 		cluster: c,
+		builder: fscache,
 	}
 
 	r.routes = []router.Route{

api/server/router/system/system_routes.go

@@ -17,6 +17,7 @@ import (
 	timetypes "github.com/docker/docker/api/types/time"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/pkg/ioutils"
+	pkgerrors "github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
@@ -75,6 +76,11 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
 	if err != nil {
 		return err
 	}
+	builderSize, err := s.builder.DiskUsage()
+	if err != nil {
+		return pkgerrors.Wrap(err, "error getting build cache usage")
+	}
+	du.BuilderSize = builderSize
 
 	return httputils.WriteJSON(w, http.StatusOK, du)
 }

api/swagger.yaml

@@ -4745,6 +4745,27 @@ paths:
           schema:
             $ref: "#/definitions/ErrorResponse"
       tags: ["Image"]
+  /build/prune:
+    post:
+      summary: "Delete builder cache"
+      produces:
+        - "application/json"
+      operationId: "BuildPrune"
+      responses:
+        200:
+          description: "No error"
+          schema:
+            type: "object"
+            properties:
+              SpaceReclaimed:
+                description: "Disk space reclaimed in bytes"
+                type: "integer"
+                format: "int64"
+        500:
+          description: "Server error"
+          schema:
+            $ref: "#/definitions/ErrorResponse"
+      tags: ["Image"]
   /images/create:
     post:
       summary: "Create an image"

api/types/types.go

@@ -489,10 +489,11 @@ type Runtime struct {
 // DiskUsage contains response of Engine API:
 // GET "/system/df"
 type DiskUsage struct {
-	LayersSize int64
-	Images     []*ImageSummary
-	Containers []*Container
-	Volumes    []*Volume
+	LayersSize  int64
+	Images      []*ImageSummary
+	Containers  []*Container
+	Volumes     []*Volume
+	BuilderSize int64
 }
 
 // ContainersPruneReport contains the response for Engine API:
@@ -516,6 +517,12 @@ type ImagesPruneReport struct {
 	SpaceReclaimed uint64
 }
 
+// BuildCachePruneReport contains the response for Engine API:
+// POST "/build/prune"
+type BuildCachePruneReport struct {
+	SpaceReclaimed uint64
+}
+
 // NetworksPruneReport contains the response for Engine API:
 // POST "/networks/prune"
 type NetworksPruneReport struct {

builder/dockerfile/builder.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/api/types"
@@ -15,6 +16,7 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/command"
 	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/builder/remotecontext"
 	"github.com/docker/docker/client/session"
 	"github.com/docker/docker/pkg/archive"
@@ -52,16 +54,22 @@ type BuildManager struct {
 	backend   builder.Backend
 	pathCache pathCache // TODO: make this persistent
 	sg        SessionGetter
+	fsCache   *fscache.FSCache
 }
 
 // NewBuildManager creates a BuildManager
-func NewBuildManager(b builder.Backend, sg SessionGetter, idMappings *idtools.IDMappings) *BuildManager {
-	return &BuildManager{
+func NewBuildManager(b builder.Backend, sg SessionGetter, fsCache *fscache.FSCache, idMappings *idtools.IDMappings) (*BuildManager, error) {
+	bm := &BuildManager{
 		backend:   b,
 		pathCache: &syncmap.Map{},
 		sg:        sg,
 		archiver:  chrootarchive.NewArchiver(idMappings),
+		fsCache:   fsCache,
 	}
+	if err := fsCache.RegisterTransport(remotecontext.ClientSessionRemote, NewClientSessionTransport()); err != nil {
+		return nil, err
+	}
+	return bm, nil
 }
 
 // Build starts a new build from a BuildConfig
@@ -75,13 +83,13 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
 	if err != nil {
 		return nil, err
 	}
-	if source != nil {
-		defer func() {
+	defer func() {
+		if source != nil {
 			if err := source.Close(); err != nil {
 				logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
 			}
-		}()
-	}
+		}
+	}()
 
 	// TODO @jhowardmsft LCOW support - this will require rework to allow both linux and Windows simultaneously.
 	// This is an interim solution to hardcode to linux if LCOW is turned on.
@@ -95,8 +103,10 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	if err := bm.initializeClientSession(ctx, cancel, config.Options); err != nil {
+	if src, err := bm.initializeClientSession(ctx, cancel, config.Options); err != nil {
 		return nil, err
+	} else if src != nil {
+		source = src
 	}
 
 	builderOptions := builderOptions{
@@ -111,20 +121,38 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
 	return newBuilder(ctx, builderOptions).build(source, dockerfile)
 }
 
-func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) error {
+func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) (builder.Source, error) {
 	if options.SessionID == "" || bm.sg == nil {
-		return nil
+		return nil, nil
 	}
 	logrus.Debug("client is session enabled")
+
+	ctx, cancelCtx := context.WithTimeout(ctx, sessionConnectTimeout)
+	defer cancelCtx()
+
 	c, err := bm.sg.Get(ctx, options.SessionID)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	go func() {
 		<-c.Context().Done()
 		cancel()
 	}()
-	return nil
+	if options.RemoteContext == remotecontext.ClientSessionRemote {
+		st := time.Now()
+		csi, err := NewClientSessionSourceIdentifier(ctx, bm.sg,
+			options.SessionID, []string{"/"})
+		if err != nil {
+			return nil, err
+		}
+		src, err := bm.fsCache.SyncFrom(ctx, csi)
+		if err != nil {
+			return nil, err
+		}
+		logrus.Debugf("sync-time: %v", time.Since(st))
+		return src, nil
+	}
+	return nil, nil
 }
 
 // builderOptions are the dependencies required by the builder

builder/dockerfile/clientsession.go

@@ -0,0 +1,78 @@
+package dockerfile
+
+import (
+	"time"
+
+	"github.com/docker/docker/builder/fscache"
+	"github.com/docker/docker/builder/remotecontext"
+	"github.com/docker/docker/client/session"
+	"github.com/docker/docker/client/session/filesync"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+)
+
+const sessionConnectTimeout = 5 * time.Second
+
+// ClientSessionTransport is a transport for copying files from docker client
+// to the daemon.
+type ClientSessionTransport struct{}
+
+// NewClientSessionTransport returns new ClientSessionTransport instance
+func NewClientSessionTransport() *ClientSessionTransport {
+	return &ClientSessionTransport{}
+}
+
+// Copy data from a remote to a destination directory.
+func (cst *ClientSessionTransport) Copy(ctx context.Context, id fscache.RemoteIdentifier, dest string, cu filesync.CacheUpdater) error {
+	csi, ok := id.(*ClientSessionSourceIdentifier)
+	if !ok {
+		return errors.New("invalid identifier for client session")
+	}
+
+	return filesync.FSSync(ctx, csi.caller, filesync.FSSendRequestOpt{
+		SrcPaths:     csi.srcPaths,
+		DestDir:      dest,
+		CacheUpdater: cu,
+	})
+}
+
+// ClientSessionSourceIdentifier is an identifier that can be used for requesting
+// files from remote client
+type ClientSessionSourceIdentifier struct {
+	srcPaths  []string
+	caller    session.Caller
+	sharedKey string
+	uuid      string
+}
+
+// NewClientSessionSourceIdentifier returns new ClientSessionSourceIdentifier instance
+func NewClientSessionSourceIdentifier(ctx context.Context, sg SessionGetter, uuid string, sources []string) (*ClientSessionSourceIdentifier, error) {
+	csi := &ClientSessionSourceIdentifier{
+		uuid:     uuid,
+		srcPaths: sources,
+	}
+	caller, err := sg.Get(ctx, uuid)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get session for %s", uuid)
+	}
+
+	csi.caller = caller
+	return csi, nil
+}
+
+// Transport returns transport identifier for remote identifier
+func (csi *ClientSessionSourceIdentifier) Transport() string {
+	return remotecontext.ClientSessionRemote
+}
+
+// SharedKey returns shared key for remote identifier. Shared key is used
+// for finding the base for a repeated transfer.
+func (csi *ClientSessionSourceIdentifier) SharedKey() string {
+	return csi.caller.SharedKey()
+}
+
+// Key returns unique key for remote identifier. Requests with same key return
+// same data.
+func (csi *ClientSessionSourceIdentifier) Key() string {
+	return csi.uuid
+}