Skip to content

Commit

Permalink
Abstract builder and implement server-side dockerfile builder
Browse files Browse the repository at this point in the history
This patch creates interfaces in builder/ for building Docker images.
It is a first step in a series of patches to remove the daemon
dependency on builder and later allow a client-side Dockerfile builder
as well as potential builder plugins.

It is needed because we cannot remove the /build API endpoint, so we
need to keep the server-side Dockerfile builder, but we also want to
reuse the same Dockerfile parser and evaluator for both server-side and
client-side.

builder/dockerfile/ and api/server/builder.go contain implementations
of those interfaces as a refactoring of the current code.

Signed-off-by: Tibor Vass <[email protected]>
  • Loading branch information
Tibor Vass committed Oct 6, 2015
1 parent f41230b commit e0ef11a
Show file tree
Hide file tree
Showing 26 changed files with 1,693 additions and 1,313 deletions.
16 changes: 11 additions & 5 deletions api/client/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"runtime"
Expand Down Expand Up @@ -131,13 +130,19 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
return fmt.Errorf("cannot canonicalize dockerfile path %s: %v", relDockerfile, err)
}

var includes = []string{"."}

excludes, err := utils.ReadDockerIgnore(path.Join(contextDir, ".dockerignore"))
if err != nil {
f, err := os.Open(filepath.Join(contextDir, ".dockerignore"))
if err != nil && !os.IsNotExist(err) {
return err
}

var excludes []string
if err == nil {
excludes, err = utils.ReadDockerIgnore(f)
if err != nil {
return err
}
}

if err := utils.ValidateContextDirectory(contextDir, excludes); err != nil {
return fmt.Errorf("Error checking context: '%s'.", err)
}
Expand All @@ -149,6 +154,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
// removed. The deamon will remove them for us, if needed, after it
// parses the Dockerfile. Ignore errors here, as they will have been
// caught by ValidateContextDirectory above.
var includes = []string{"."}
keepThem1, _ := fileutils.Matches(".dockerignore", excludes)
keepThem2, _ := fileutils.Matches(relDockerfile, excludes)
if keepThem1 || keepThem2 {
Expand Down
122 changes: 98 additions & 24 deletions api/server/router/local/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,18 @@ import (
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/server/httputils"
"github.com/docker/docker/api/types"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile"
"github.com/docker/docker/cliconfig"
"github.com/docker/docker/daemon/daemonbuilder"
"github.com/docker/docker/graph"
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/ulimit"
"github.com/docker/docker/registry"
"github.com/docker/docker/runconfig"
"github.com/docker/docker/utils"
"golang.org/x/net/context"
Expand Down Expand Up @@ -56,13 +61,18 @@ func (s *router) postCommit(ctx context.Context, w http.ResponseWriter, r *http.
Config: c,
}

imgID, err := dockerfile.Commit(cname, s.daemon, commitCfg)
container, err := s.daemon.Get(cname)
if err != nil {
return err
}

imgID, err := dockerfile.Commit(container, s.daemon, commitCfg)
if err != nil {
return err
}

return httputils.WriteJSON(w, http.StatusCreated, &types.ContainerCommitResponse{
ID: imgID,
ID: string(imgID),
})
}

Expand Down Expand Up @@ -125,7 +135,7 @@ func (s *router) postImagesCreate(ctx context.Context, w http.ResponseWriter, r
// generated from the download to be available to the output
// stream processing below
var newConfig *runconfig.Config
newConfig, err = dockerfile.BuildFromConfig(s.daemon, &runconfig.Config{}, r.Form["changes"])
newConfig, err = dockerfile.BuildFromConfig(&runconfig.Config{}, r.Form["changes"])
if err != nil {
return err
}
Expand Down Expand Up @@ -269,7 +279,7 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
var (
authConfigs = map[string]cliconfig.AuthConfig{}
authConfigsEncoded = r.Header.Get("X-Registry-Config")
buildConfig = dockerfile.NewBuildConfig()
buildConfig = &dockerfile.Config{}
)

if authConfigsEncoded != "" {
Expand All @@ -284,6 +294,21 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
w.Header().Set("Content-Type", "application/json")

version := httputils.VersionFromContext(ctx)
output := ioutils.NewWriteFlusher(w)
sf := streamformatter.NewJSONStreamFormatter()
errf := func(err error) error {
// Do not write the error in the http output if it's still empty.
// This prevents from writing a 200(OK) when there is an interal error.
if !output.Flushed() {
return err
}
_, err = w.Write(sf.FormatError(errors.New(utils.GetErrorMessage(err))))
if err != nil {
logrus.Warnf("could not write error response: %v", err)
}
return nil
}

if httputils.BoolValue(r, "forcerm") && version.GreaterThanOrEqualTo("1.12") {
buildConfig.Remove = true
} else if r.FormValue("rm") == "" && version.GreaterThanOrEqualTo("1.12") {
Expand All @@ -295,17 +320,22 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
buildConfig.Pull = true
}

output := ioutils.NewWriteFlusher(w)
buildConfig.Stdout = output
buildConfig.Context = r.Body
repoName, tag := parsers.ParseRepositoryTag(r.FormValue("t"))
if repoName != "" {
if err := registry.ValidateRepositoryName(repoName); err != nil {
return errf(err)
}
if len(tag) > 0 {
if err := tags.ValidateTagName(tag); err != nil {
return errf(err)
}
}
}

buildConfig.RemoteURL = r.FormValue("remote")
buildConfig.DockerfileName = r.FormValue("dockerfile")
buildConfig.RepoName = r.FormValue("t")
buildConfig.SuppressOutput = httputils.BoolValue(r, "q")
buildConfig.NoCache = httputils.BoolValue(r, "nocache")
buildConfig.Verbose = !httputils.BoolValue(r, "q")
buildConfig.UseCache = !httputils.BoolValue(r, "nocache")
buildConfig.ForceRemove = httputils.BoolValue(r, "forcerm")
buildConfig.AuthConfigs = authConfigs
buildConfig.MemorySwap = httputils.Int64ValueOrZero(r, "memswap")
buildConfig.Memory = httputils.Int64ValueOrZero(r, "memory")
buildConfig.CPUShares = httputils.Int64ValueOrZero(r, "cpushares")
Expand All @@ -319,7 +349,7 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
ulimitsJSON := r.FormValue("ulimits")
if ulimitsJSON != "" {
if err := json.NewDecoder(strings.NewReader(ulimitsJSON)).Decode(&buildUlimits); err != nil {
return err
return errf(err)
}
buildConfig.Ulimits = buildUlimits
}
Expand All @@ -328,12 +358,50 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
buildArgsJSON := r.FormValue("buildargs")
if buildArgsJSON != "" {
if err := json.NewDecoder(strings.NewReader(buildArgsJSON)).Decode(&buildArgs); err != nil {
return err
return errf(err)
}
buildConfig.BuildArgs = buildArgs
}
buildConfig.BuildArgs = buildArgs

// Job cancellation. Note: not all job types support this.
remoteURL := r.FormValue("remote")

// Currently, only used if context is from a remote url.
// The field `In` is set by DetectContextFromRemoteURL.
// Look at code in DetectContextFromRemoteURL for more information.
pReader := &progressreader.Config{
// TODO: make progressreader streamformatter-agnostic
Out: output,
Formatter: sf,
Size: r.ContentLength,
NewLines: true,
ID: "Downloading context",
Action: remoteURL,
}

var (
context builder.ModifiableContext
dockerfileName string
err error
)
context, dockerfileName, err = daemonbuilder.DetectContextFromRemoteURL(r.Body, remoteURL, pReader)
if err != nil {
return errf(err)
}
defer func() {
if err := context.Close(); err != nil {
logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
}
}()

docker := daemonbuilder.Docker{s.daemon, output, authConfigs}

b, err := dockerfile.NewBuilder(buildConfig, docker, builder.DockerIgnoreContext{context}, nil)
if err != nil {
return errf(err)
}
b.Stdout = &streamformatter.StdoutFormatter{Writer: output, StreamFormatter: sf}
b.Stderr = &streamformatter.StderrFormatter{Writer: output, StreamFormatter: sf}

if closeNotifier, ok := w.(http.CloseNotifier); ok {
finished := make(chan struct{})
defer close(finished)
Expand All @@ -342,20 +410,26 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
case <-finished:
case <-closeNotifier.CloseNotify():
logrus.Infof("Client disconnected, cancelling job: build")
buildConfig.Cancel()
b.Cancel()
}
}()
}

if err := dockerfile.Build(s.daemon, buildConfig); err != nil {
// Do not write the error in the http output if it's still empty.
// This prevents from writing a 200(OK) when there is an interal error.
if !output.Flushed() {
return err
if len(dockerfileName) > 0 {
b.DockerfileName = dockerfileName
}

imgID, err := b.Build()
if err != nil {
return errf(err)
}

if repoName != "" {
if err := s.daemon.Repositories().Tag(repoName, tag, string(imgID), true); err != nil {
return errf(err)
}
sf := streamformatter.NewJSONStreamFormatter()
w.Write(sf.FormatError(errors.New(utils.GetErrorMessage(err))))
}

return nil
}

Expand Down
139 changes: 139 additions & 0 deletions builder/builder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
// Package builder defines interfaces for any Docker builder to implement.
//
// Historically, only server-side Dockerfile interpreters existed.
// This package allows for other implementations of Docker builders.
package builder

import (
"io"
"os"

// TODO: remove dependency on daemon
"github.com/docker/docker/daemon"
"github.com/docker/docker/image"
"github.com/docker/docker/runconfig"
)

// Builder abstracts a Docker builder whose only purpose is to build a Docker image referenced by an imageID.
type Builder interface {
// Build builds a Docker image referenced by an imageID string.
//
// Note: Tagging an image should not be done by a Builder, it should instead be done
// by the caller.
//
// TODO: make this return a reference instead of string
Build() (imageID string)
}

// Context represents a file system tree.
type Context interface {
// Close allows to signal that the filesystem tree won't be used anymore.
// For Context implementations using a temporary directory, it is recommended to
// delete the temporary directory in Close().
Close() error
// Stat returns an entry corresponding to path if any.
// It is recommended to return an error if path was not found.
Stat(path string) (FileInfo, error)
// Open opens path from the context and returns a readable stream of it.
Open(path string) (io.ReadCloser, error)
// Walk walks the tree of the context with the function passed to it.
Walk(root string, walkFn WalkFunc) error
}

// WalkFunc is the type of the function called for each file or directory visited by Context.Walk().
type WalkFunc func(path string, fi FileInfo, err error) error

// ModifiableContext represents a modifiable Context.
// TODO: remove this interface once we can get rid of Remove()
type ModifiableContext interface {
Context
// Remove deletes the entry specified by `path`.
// It is usual for directory entries to delete all its subentries.
Remove(path string) error
}

// FileInfo extends os.FileInfo to allow retrieving an absolute path to the file.
// TODO: remove this interface once pkg/archive exposes a walk function that Context can use.
type FileInfo interface {
os.FileInfo
Path() string
}

// PathFileInfo is a convenience struct that implements the FileInfo interface.
type PathFileInfo struct {
os.FileInfo
// FilePath holds the absolute path to the file.
FilePath string
}

// Path returns the absolute path to the file.
func (fi PathFileInfo) Path() string {
return fi.FilePath
}

// Hashed defines an extra method intended for implementations of os.FileInfo.
type Hashed interface {
// Hash returns the hash of a file.
Hash() string
SetHash(string)
}

// HashedFileInfo is a convenient struct that augments FileInfo with a field.
type HashedFileInfo struct {
FileInfo
// FileHash represents the hash of a file.
FileHash string
}

// Hash returns the hash of a file.
func (fi HashedFileInfo) Hash() string {
return fi.FileHash
}

// SetHash sets the hash of a file.
func (fi *HashedFileInfo) SetHash(h string) {
fi.FileHash = h
}

// Docker abstracts calls to a Docker Daemon.
type Docker interface {
// TODO: use digest reference instead of name

// LookupImage looks up a Docker image referenced by `name`.
LookupImage(name string) (*image.Image, error)
// Pull tells Docker to pull image referenced by `name`.
Pull(name string) (*image.Image, error)

// TODO: move daemon.Container to its own package

// Container looks up a Docker container referenced by `id`.
Container(id string) (*daemon.Container, error)
// Create creates a new Docker container and returns potential warnings
// TODO: put warnings in the error
Create(*runconfig.Config, *runconfig.HostConfig) (*daemon.Container, []string, error)
// Remove removes a container specified by `id`.
Remove(id string, cfg *daemon.ContainerRmConfig) error
// Commit creates a new Docker image from an existing Docker container.
Commit(*daemon.Container, *daemon.ContainerCommitConfig) (*image.Image, error)
// Copy copies/extracts a source FileInfo to a destination path inside a container
// specified by a container object.
// TODO: make an Extract method instead of passing `decompress`
// TODO: do not pass a FileInfo, instead refactor the archive package to export a Walk function that can be used
// with Context.Walk
Copy(c *daemon.Container, destPath string, src FileInfo, decompress bool) error

// Retain retains an image avoiding it to be removed or overwritten until a corresponding Release() call.
// TODO: remove
Retain(sessionID, imgID string)
// Release releases a list of images that were retained for the time of a build.
// TODO: remove
Release(sessionID string, activeImages []string)
}

// ImageCache abstracts an image cache store.
// (parent image, child runconfig) -> child image
type ImageCache interface {
// GetCachedImage returns a reference to a cached image whose parent equals `parent`
// and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error.
GetCachedImage(parentID string, cfg *runconfig.Config) (imageID string, err error)
}
Loading

0 comments on commit e0ef11a

Please sign in to comment.