Merge pull request moby#33241 from Microsoft/jjh/multi-layerstore

LCOW: Support most operations excluding remote filesystem

api/common.go

@@ -126,7 +126,7 @@ func MatchesContentType(contentType, expectedType string) bool {
 // LoadOrCreateTrustKey attempts to load the libtrust key at the given path,
 // otherwise generates a new one
 func LoadOrCreateTrustKey(trustKeyPath string) (libtrust.PrivateKey, error) {
-	err := system.MkdirAll(filepath.Dir(trustKeyPath), 0700)
+	err := system.MkdirAll(filepath.Dir(trustKeyPath), 0700, "")
 	if err != nil {
 		return nil, err
 	}

api/server/backend/build/backend.go

@@ -17,7 +17,7 @@ import (
 // ImageComponent provides an interface for working with images
 type ImageComponent interface {
 	SquashImage(from string, to string) (string, error)
-	TagImageWithReference(image.ID, reference.Named) error
+	TagImageWithReference(image.ID, string, reference.Named) error
 }
 
 // Backend provides build functionality to the API router

api/server/backend/build/tag.go

@@ -3,9 +3,11 @@ package build
 import (
 	"fmt"
 	"io"
+	"runtime"
 
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/image"
+	"github.com/docker/docker/pkg/system"
 	"github.com/pkg/errors"
 )
 
@@ -33,7 +35,12 @@ func NewTagger(backend ImageComponent, stdout io.Writer, names []string) (*Tagge
 // TagImages creates image tags for the imageID
 func (bt *Tagger) TagImages(imageID image.ID) error {
 	for _, rt := range bt.repoAndTags {
-		if err := bt.imageComponent.TagImageWithReference(imageID, rt); err != nil {
+		// TODO @jhowardmsft LCOW support. Will need revisiting.
+		platform := runtime.GOOS
+		if platform == "windows" && system.LCOWSupported() {
+			platform = "linux"
+		}
+		if err := bt.imageComponent.TagImageWithReference(imageID, platform, rt); err != nil {
 			return err
 		}
 		fmt.Fprintf(bt.stdout, "Successfully tagged %s\n", reference.FamiliarString(rt))

api/server/router/image/backend.go

@@ -35,12 +35,12 @@ type imageBackend interface {
 
 type importExportBackend interface {
 	LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error
-	ImportImage(src string, repository, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error
+	ImportImage(src string, repository, platform string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error
 	ExportImage(names []string, outStream io.Writer) error
 }
 
 type registryBackend interface {
-	PullImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
+	PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
 	PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
 	SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, authConfig *types.AuthConfig, metaHeaders map[string][]string) (*registry.SearchResults, error)
 }

api/server/router/image/image_routes.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"runtime"
 	"strconv"
 	"strings"
 
@@ -17,6 +18,7 @@ import (
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/registry"
 	"golang.org/x/net/context"
 )
@@ -85,6 +87,41 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
 	)
 	defer output.Close()
 
+	// TODO @jhowardmsft LCOW Support: Eventually we will need an API change
+	// so that platform comes from (for example) r.Form.Get("platform"). For
+	// the initial implementation, we assume that the platform is the
+	// runtime OS of the host. It will also need a validation function such
+	// as below which should be called after getting it from the API.
+	//
+	// Ensures the requested platform is valid and normalized
+	//func validatePlatform(req string) (string, error) {
+	//	req = strings.ToLower(req)
+	//	if req == "" {
+	//		req = runtime.GOOS // default to host platform
+	//	}
+	//	valid := []string{runtime.GOOS}
+	//
+	//	if runtime.GOOS == "windows" && system.LCOWSupported() {
+	//		valid = append(valid, "linux")
+	//	}
+	//
+	//	for _, item := range valid {
+	//		if req == item {
+	//			return req, nil
+	//		}
+	//	}
+	//	return "", fmt.Errorf("invalid platform requested: %s", req)
+	//}
+	//
+	// And in the call-site:
+	//	if platform, err = validatePlatform(platform); err != nil {
+	//		return err
+	//	}
+	platform := runtime.GOOS
+	if platform == "windows" && system.LCOWSupported() {
+		platform = "linux"
+	}
+
 	w.Header().Set("Content-Type", "application/json")
 
 	if image != "" { //pull
@@ -106,13 +143,13 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
 			}
 		}
 
-		err = s.backend.PullImage(ctx, image, tag, metaHeaders, authConfig, output)
+		err = s.backend.PullImage(ctx, image, tag, platform, metaHeaders, authConfig, output)
 	} else { //import
 		src := r.Form.Get("fromSrc")
 		// 'err' MUST NOT be defined within this block, we need any error
 		// generated from the download to be available to the output
 		// stream processing below
-		err = s.backend.ImportImage(src, repo, tag, message, r.Body, output, r.Form["changes"])
+		err = s.backend.ImportImage(src, repo, platform, tag, message, r.Body, output, r.Form["changes"])
 	}
 	if err != nil {
 		if !output.Flushed() {

api/types/backend/build.go

@@ -40,4 +40,5 @@ type GetImageAndLayerOptions struct {
 	PullOption PullOption
 	AuthConfig map[string]types.AuthConfig
 	Output     io.Writer
+	Platform   string
 }

api/types/client.go

@@ -178,6 +178,10 @@ type ImageBuildOptions struct {
 	SecurityOpt []string
 	ExtraHosts  []string // List of extra hosts
 	Target      string
+
+	// TODO @jhowardmsft LCOW Support: This will require extending to include
+	// `Platform string`, but is ommited for now as it's hard-coded temporarily
+	// to avoid API changes.
 }
 
 // ImageBuildResponse holds information

api/types/configs.go

@@ -16,6 +16,7 @@ type ContainerCreateConfig struct {
 	HostConfig       *container.HostConfig
 	NetworkingConfig *network.NetworkingConfig
 	AdjustCPUShares  bool
+	Platform         string
 }
 
 // ContainerRmConfig holds arguments for the container remove

api/types/types.go

@@ -320,6 +320,7 @@ type ContainerJSONBase struct {
 	Name            string
 	RestartCount    int
 	Driver          string
+	Platform        string
 	MountLabel      string
 	ProcessLabel    string
 	AppArmorProfile string

builder/builder.go

@@ -43,7 +43,7 @@ type Backend interface {
 	// ContainerCreateWorkdir creates the workdir
 	ContainerCreateWorkdir(containerID string) error
 
-	CreateImage(config []byte, parent string) (Image, error)
+	CreateImage(config []byte, parent string, platform string) (Image, error)
 
 	ImageCacheBuilder
 }
@@ -78,7 +78,7 @@ type Result struct {
 // ImageCacheBuilder represents a generator for stateful image cache.
 type ImageCacheBuilder interface {
 	// MakeImageCache creates a stateful image cache.
-	MakeImageCache(cacheFrom []string) ImageCache
+	MakeImageCache(cacheFrom []string, platform string) ImageCache
 }
 
 // ImageCache abstracts an image cache.
@@ -100,6 +100,6 @@ type Image interface {
 type ReleaseableLayer interface {
 	Release() error
 	Mount() (string, error)
-	Commit() (ReleaseableLayer, error)
+	Commit(platform string) (ReleaseableLayer, error)
 	DiffID() layer.DiffID
 }

builder/dockerfile/builder.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"runtime"
 	"strings"
 
 	"github.com/Sirupsen/logrus"
@@ -20,6 +21,7 @@ import (
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/pkg/system"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"golang.org/x/sync/syncmap"
@@ -73,13 +75,24 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
 		}()
 	}
 
+	// TODO @jhowardmsft LCOW support - this will require rework to allow both linux and Windows simultaneously.
+	// This is an interim solution to hardcode to linux if LCOW is turned on.
+	if dockerfile.Platform == "" {
+		dockerfile.Platform = runtime.GOOS
+		if dockerfile.Platform == "windows" && system.LCOWSupported() {
+			dockerfile.Platform = "linux"
+		}
+	}
+
 	builderOptions := builderOptions{
 		Options:        config.Options,
 		ProgressWriter: config.ProgressWriter,
 		Backend:        bm.backend,
 		PathCache:      bm.pathCache,
 		Archiver:       bm.archiver,
+		Platform:       dockerfile.Platform,
 	}
+
 	return newBuilder(ctx, builderOptions).build(source, dockerfile)
 }
 
@@ -90,6 +103,7 @@ type builderOptions struct {
 	ProgressWriter backend.ProgressWriter
 	PathCache      pathCache
 	Archiver       *archive.Archiver
+	Platform       string
 }
 
 // Builder is a Dockerfile builder
@@ -113,14 +127,32 @@ type Builder struct {
 	pathCache        pathCache
 	containerManager *containerManager
 	imageProber      ImageProber
+
+	// TODO @jhowardmft LCOW Support. This will be moved to options at a later
+	// stage, however that cannot be done now as it affects the public API
+	// if it were.
+	platform string
 }
 
 // newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
+// TODO @jhowardmsft LCOW support: Eventually platform can be moved into the builder
+// options, however, that would be an API change as it shares types.ImageBuildOptions.
 func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 	config := options.Options
 	if config == nil {
 		config = new(types.ImageBuildOptions)
 	}
+
+	// @jhowardmsft LCOW Support. For the time being, this is interim. Eventually
+	// will be moved to types.ImageBuildOptions, but it can't for now as that would
+	// be an API change.
+	if options.Platform == "" {
+		options.Platform = runtime.GOOS
+	}
+	if options.Platform == "windows" && system.LCOWSupported() {
+		options.Platform = "linux"
+	}
+
 	b := &Builder{
 		clientCtx:        clientCtx,
 		options:          config,
@@ -134,9 +166,11 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 		buildStages:      newBuildStages(),
 		imageSources:     newImageSources(clientCtx, options),
 		pathCache:        options.PathCache,
-		imageProber:      newImageProber(options.Backend, config.CacheFrom, config.NoCache),
+		imageProber:      newImageProber(options.Backend, config.CacheFrom, options.Platform, config.NoCache),
 		containerManager: newContainerManager(options.Backend),
+		platform:         options.Platform,
 	}
+
 	return b
 }
 
@@ -267,6 +301,17 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
 		return nil, err
 	}
 
+	// TODO @jhowardmsft LCOW support. For now, if LCOW enabled, switch to linux.
+	// Also explicitly set the platform. Ultimately this will be in the builder
+	// options, but we can't do that yet as it would change the API.
+	if dockerfile.Platform == "" {
+		dockerfile.Platform = runtime.GOOS
+	}
+	if dockerfile.Platform == "windows" && system.LCOWSupported() {
+		dockerfile.Platform = "linux"
+	}
+	b.platform = dockerfile.Platform
+
 	// ensure that the commands are valid
 	for _, n := range dockerfile.AST.Children {
 		if !validCommitCommands[n.Value] {

builder/dockerfile/builder_unix.go

@@ -2,4 +2,6 @@
 
 package dockerfile
 
-var defaultShell = []string{"/bin/sh", "-c"}
+func defaultShellForPlatform(platform string) []string {
+	return []string{"/bin/sh", "-c"}
+}

builder/dockerfile/builder_windows.go

@@ -1,3 +1,8 @@
 package dockerfile
 
-var defaultShell = []string{"cmd", "/S", "/C"}
+func defaultShellForPlatform(platform string) []string {
+	if platform == "linux" {
+		return []string{"/bin/sh", "-c"}
+	}
+	return []string{"cmd", "/S", "/C"}
+}

builder/dockerfile/containerbackend.go

@@ -28,10 +28,11 @@ func newContainerManager(docker builder.ExecBackend) *containerManager {
 }
 
 // Create a container
-func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig) (container.ContainerCreateCreatedBody, error) {
+func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig, platform string) (container.ContainerCreateCreatedBody, error) {
 	container, err := c.backend.ContainerCreate(types.ContainerCreateConfig{
 		Config:     runConfig,
 		HostConfig: hostConfig,
+		Platform:   platform,
 	})
 	if err != nil {
 		return container, err