Merge pull request moby#22641 from cpuguy83/build_finalization

Adds ability to flatten image after build

api/server/router/build/build_routes.go

@@ -54,6 +54,7 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 	options.NetworkMode = r.FormValue("networkmode")
 	options.Tags = r.Form["t"]
 	options.SecurityOpt = r.Form["securityopt"]
+	options.Squash = httputils.BoolValue(r, "squash")
 
 	if r.Form.Get("shmsize") != "" {
 		shmSize, err := strconv.ParseInt(r.Form.Get("shmsize"), 10, 64)

builder/builder.go

@@ -135,9 +135,15 @@ type Backend interface {
 	// TODO: make an Extract method instead of passing `decompress`
 	// TODO: do not pass a FileInfo, instead refactor the archive package to export a Walk function that can be used
 	// with Context.Walk
-	//ContainerCopy(name string, res string) (io.ReadCloser, error)
+	// ContainerCopy(name string, res string) (io.ReadCloser, error)
 	// TODO: use copyBackend api
 	CopyOnBuild(containerID string, destPath string, src FileInfo, decompress bool) error
+
+	// HasExperimental checks if the backend supports experimental features
+	HasExperimental() bool
+
+	// SquashImage squashes the fs layers from the provided image down to the specified `to` image
+	SquashImage(from string, to string) (string, error)
 }
 
 // Image represents a Docker image used by the builder.

builder/dockerfile/builder.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/Sirupsen/logrus"
+	apierrors "github.com/docker/docker/api/errors"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
@@ -18,6 +19,7 @@ import (
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/reference"
+	perrors "github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
@@ -77,6 +79,7 @@ type Builder struct {
 	id string
 
 	imageCache builder.ImageCache
+	from       builder.Image
 }
 
 // BuildManager implements builder.Backend and is shared across all Builder objects.
@@ -91,6 +94,9 @@ func NewBuildManager(b builder.Backend) (bm *BuildManager) {
 
 // BuildFromContext builds a new image from a given context.
 func (bm *BuildManager) BuildFromContext(ctx context.Context, src io.ReadCloser, remote string, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error) {
+	if buildOptions.Squash && !bm.backend.HasExperimental() {
+		return "", apierrors.NewBadRequestError(errors.New("squash is only supported with experimental mode"))
+	}
 	buildContext, dockerfileName, err := builder.DetectContextFromRemoteURL(src, remote, pg.ProgressReaderFunc)
 	if err != nil {
 		return "", err
@@ -100,6 +106,7 @@ func (bm *BuildManager) BuildFromContext(ctx context.Context, src io.ReadCloser,
 			logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
 		}
 	}()
+
 	if len(dockerfileName) > 0 {
 		buildOptions.Dockerfile = dockerfileName
 	}
@@ -287,6 +294,17 @@ func (b *Builder) build(stdout io.Writer, stderr io.Writer, out io.Writer) (stri
 		return "", fmt.Errorf("No image was generated. Is your Dockerfile empty?")
 	}
 
+	if b.options.Squash {
+		var fromID string
+		if b.from != nil {
+			fromID = b.from.ImageID()
+		}
+		b.image, err = b.docker.SquashImage(b.image, fromID)
+		if err != nil {
+			return "", perrors.Wrap(err, "error squashing image")
+		}
+	}
+
 	imageID := image.ID(b.image)
 	for _, rt := range repoAndTags {
 		if err := b.docker.TagImageWithReference(imageID, rt); err != nil {

builder/dockerfile/dispatchers.go

@@ -221,6 +221,7 @@ func from(b *Builder, args []string, attributes map[string]bool, original string
 			}
 		}
 	}
+	b.from = image
 
 	return b.processImageFrom(image)
 }

cli/command/image/build.go

@@ -59,6 +59,7 @@ type buildOptions struct {
 	compress       bool
 	securityOpt    []string
 	networkMode    string
+	squash         bool
 }
 
 // NewBuildCommand creates a new `docker build` command
@@ -110,6 +111,10 @@ func NewBuildCommand(dockerCli *command.DockerCli) *cobra.Command {
 
 	command.AddTrustedFlags(flags, true)
 
+	if dockerCli.HasExperimental() {
+		flags.BoolVar(&options.squash, "squash", false, "Squash newly built layers into a single new layer")
+	}
+
 	return cmd
 }
 
@@ -305,6 +310,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 		CacheFrom:      options.cacheFrom,
 		SecurityOpt:    options.securityOpt,
 		NetworkMode:    options.networkMode,
+		Squash:         options.squash,
 	}
 
 	response, err := dockerCli.Client().ImageBuild(ctx, body, buildOptions)

daemon/graphdriver/aufs/aufs.go

@@ -74,6 +74,7 @@ type Driver struct {
 	ctr           *graphdriver.RefCounter
 	pathCacheLock sync.Mutex
 	pathCache     map[string]string
+	naiveDiff     graphdriver.DiffDriver
 }
 
 // Init returns a new AUFS driver.
@@ -137,6 +138,8 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 			return nil, err
 		}
 	}
+
+	a.naiveDiff = graphdriver.NewNaiveDiffDriver(a, uidMaps, gidMaps)
 	return a, nil
 }
 
@@ -225,7 +228,7 @@ func (a *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str
 	defer f.Close()
 
 	if parent != "" {
-		ids, err := getParentIds(a.rootPath(), parent)
+		ids, err := getParentIDs(a.rootPath(), parent)
 		if err != nil {
 			return err
 		}
@@ -427,9 +430,22 @@ func (a *Driver) Put(id string) error {
 	return err
 }
 
+// isParent returns if the passed in parent is the direct parent of the passed in layer
+func (a *Driver) isParent(id, parent string) bool {
+	parents, _ := getParentIDs(a.rootPath(), id)
+	if parent == "" && len(parents) > 0 {
+		return false
+	}
+	return !(len(parents) > 0 && parent != parents[0])
+}
+
 // Diff produces an archive of the changes between the specified
 // layer and its parent layer which may be "".
 func (a *Driver) Diff(id, parent string) (io.ReadCloser, error) {
+	if !a.isParent(id, parent) {
+		return a.naiveDiff.Diff(id, parent)
+	}
+
 	// AUFS doesn't need the parent layer to produce a diff.
 	return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
 		Compression:     archive.Uncompressed,
@@ -465,6 +481,9 @@ func (a *Driver) applyDiff(id string, diff io.Reader) error {
 // and its parent and returns the size in bytes of the changes
 // relative to its base filesystem directory.
 func (a *Driver) DiffSize(id, parent string) (size int64, err error) {
+	if !a.isParent(id, parent) {
+		return a.naiveDiff.DiffSize(id, parent)
+	}
 	// AUFS doesn't need the parent layer to calculate the diff size.
 	return directory.Size(path.Join(a.rootPath(), "diff", id))
 }
@@ -473,7 +492,11 @@ func (a *Driver) DiffSize(id, parent string) (size int64, err error) {
 // layer with the specified id and parent, returning the size of the
 // new layer in bytes.
 func (a *Driver) ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) {
-	// AUFS doesn't need the parent id to apply the diff.
+	if !a.isParent(id, parent) {
+		return a.naiveDiff.ApplyDiff(id, parent, diff)
+	}
+
+	// AUFS doesn't need the parent id to apply the diff if it is the direct parent.
 	if err = a.applyDiff(id, diff); err != nil {
 		return
 	}
@@ -484,6 +507,10 @@ func (a *Driver) ApplyDiff(id, parent string, diff io.Reader) (size int64, err e
 // Changes produces a list of changes between the specified layer
 // and its parent layer. If parent is "", then all changes will be ADD changes.
 func (a *Driver) Changes(id, parent string) ([]archive.Change, error) {
+	if !a.isParent(id, parent) {
+		return a.naiveDiff.Changes(id, parent)
+	}
+
 	// AUFS doesn't have snapshots, so we need to get changes from all parent
 	// layers.
 	layers, err := a.getParentLayerPaths(id)
@@ -494,7 +521,7 @@ func (a *Driver) Changes(id, parent string) ([]archive.Change, error) {
 }
 
 func (a *Driver) getParentLayerPaths(id string) ([]string, error) {
-	parentIds, err := getParentIds(a.rootPath(), id)
+	parentIds, err := getParentIDs(a.rootPath(), id)
 	if err != nil {
 		return nil, err
 	}

daemon/graphdriver/aufs/aufs_test.go

@@ -424,7 +424,7 @@ func TestChanges(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	changes, err = d.Changes("3", "")
+	changes, err = d.Changes("3", "2")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -530,7 +530,7 @@ func TestChildDiffSize(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	diffSize, err = d.DiffSize("2", "")
+	diffSize, err = d.DiffSize("2", "1")
 	if err != nil {
 		t.Fatal(err)
 	}

daemon/graphdriver/aufs/dirs.go

@@ -29,7 +29,7 @@ func loadIds(root string) ([]string, error) {
 //
 // If there are no lines in the file then the id has no parent
 // and an empty slice is returned.
-func getParentIds(root, id string) ([]string, error) {
+func getParentIDs(root, id string) ([]string, error) {
 	f, err := os.Open(path.Join(root, "layers", id))
 	if err != nil {
 		return nil, err

daemon/graphdriver/driver.go

@@ -78,9 +78,8 @@ type ProtoDriver interface {
 	Cleanup() error
 }
 
-// Driver is the interface for layered/snapshot file system drivers.
-type Driver interface {
-	ProtoDriver
+// DiffDriver is the interface to use to implement graph diffs
+type DiffDriver interface {
 	// Diff produces an archive of the changes between the specified
 	// layer and its parent layer which may be "".
 	Diff(id, parent string) (io.ReadCloser, error)
@@ -98,6 +97,12 @@ type Driver interface {
 	DiffSize(id, parent string) (size int64, err error)
 }
 
+// Driver is the interface for layered/snapshot file system drivers.
+type Driver interface {
+	ProtoDriver
+	DiffDriver
+}
+
 // DiffGetterDriver is the interface for layered file system drivers that
 // provide a specialized function for getting file contents for tar-split.
 type DiffGetterDriver interface {

daemon/graphdriver/overlay2/overlay.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
@@ -44,7 +45,7 @@ var (
 
 // Each container/image has at least a "diff" directory and "link" file.
 // If there is also a "lower" file when there are diff layers
-// below  as well as "merged" and "work" directories. The "diff" directory
+// below as well as "merged" and "work" directories. The "diff" directory
 // has the upper layer of the overlay and is used to capture any
 // changes to the layer. The "lower" file contains all the lower layer
 // mounts separated by ":" and ordered from uppermost to lowermost
@@ -86,12 +87,13 @@ type overlayOptions struct {
 
 // Driver contains information about the home directory and the list of active mounts that are created using this driver.
 type Driver struct {
-	home     string
-	uidMaps  []idtools.IDMap
-	gidMaps  []idtools.IDMap
-	ctr      *graphdriver.RefCounter
-	quotaCtl *quota.Control
-	options  overlayOptions
+	home      string
+	uidMaps   []idtools.IDMap
+	gidMaps   []idtools.IDMap
+	ctr       *graphdriver.RefCounter
+	quotaCtl  *quota.Control
+	options   overlayOptions
+	naiveDiff graphdriver.DiffDriver
 }
 
 var (
@@ -163,6 +165,8 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		ctr:     graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)),
 	}
 
+	d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps)
+
 	if backingFs == "xfs" {
 		// Try to enable project quota support over xfs.
 		if d.quotaCtl, err = quota.NewControl(home); err == nil {
@@ -525,7 +529,7 @@ func (d *Driver) Put(id string) error {
 		return nil
 	}
 	if err := syscall.Unmount(mountpoint, 0); err != nil {
-		logrus.Debugf("Failed to unmount %s overlay: %v", id, err)
+		logrus.Debugf("Failed to unmount %s overlay: %s - %v", id, mountpoint, err)
 	}
 	return nil
 }
@@ -536,8 +540,33 @@ func (d *Driver) Exists(id string) bool {
 	return err == nil
 }
 
+// isParent returns if the passed in parent is the direct parent of the passed in layer
+func (d *Driver) isParent(id, parent string) bool {
+	lowers, err := d.getLowerDirs(id)
+	if err != nil {
+		return false
+	}
+	if parent == "" && len(lowers) > 0 {
+		return false
+	}
+
+	parentDir := d.dir(parent)
+	var ld string
+	if len(lowers) > 0 {
+		ld = filepath.Dir(lowers[0])
+	}
+	if ld == "" && parent == "" {
+		return true
+	}
+	return ld == parentDir
+}
+
 // ApplyDiff applies the new layer into a root
 func (d *Driver) ApplyDiff(id string, parent string, diff io.Reader) (size int64, err error) {
+	if !d.isParent(id, parent) {
+		return d.naiveDiff.ApplyDiff(id, parent, diff)
+	}
+
 	applyDir := d.getDiffPath(id)
 
 	logrus.Debugf("Applying tar in %s", applyDir)
@@ -563,12 +592,19 @@ func (d *Driver) getDiffPath(id string) string {
 // and its parent and returns the size in bytes of the changes
 // relative to its base filesystem directory.
 func (d *Driver) DiffSize(id, parent string) (size int64, err error) {
+	if !d.isParent(id, parent) {
+		return d.naiveDiff.DiffSize(id, parent)
+	}
 	return directory.Size(d.getDiffPath(id))
 }
 
 // Diff produces an archive of the changes between the specified
 // layer and its parent layer which may be "".
 func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) {
+	if !d.isParent(id, parent) {
+		return d.naiveDiff.Diff(id, parent)
+	}
+
 	diffPath := d.getDiffPath(id)
 	logrus.Debugf("Tar with options on %s", diffPath)
 	return archive.TarWithOptions(diffPath, &archive.TarOptions{
@@ -582,6 +618,9 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) {
 // Changes produces a list of changes between the specified layer
 // and its parent layer. If parent is "", then all changes will be ADD changes.
 func (d *Driver) Changes(id, parent string) ([]archive.Change, error) {
+	if !d.isParent(id, parent) {
+		return d.naiveDiff.Changes(id, parent)
+	}
 	// Overlay doesn't have snapshots, so we need to get changes from all parent
 	// layers.
 	diffPath := d.getDiffPath(id)