From 4352da7803d182a6013a5238ce20a7c749db979a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 18 Nov 2015 14:20:54 -0800 Subject: [PATCH] Update daemon and docker core to use new content addressable storage Add distribution package for managing pulls and pushes. This is based on the old code in the graph package, with major changes to work with the new image/layer model. Add v1 migration code. Update registry, api/*, and daemon packages to use the reference package's types where applicable. Update daemon package to use image/layer/tag stores instead of the graph package Signed-off-by: Aaron Lehmann Signed-off-by: Tonis Tiigi --- api/client/build.go | 55 +- api/client/commit.go | 28 +- api/client/create.go | 65 ++- api/client/images.go | 34 +- api/client/import.go | 9 +- api/client/ps/custom.go | 4 +- api/client/pull.go | 43 +- api/client/push.go | 20 +- api/client/search.go | 6 +- api/client/tag.go | 27 +- api/client/trust.go | 35 +- api/server/router/local/image.go | 143 +++-- builder/builder.go | 2 +- builder/dockerfile/builder.go | 4 +- builder/dockerfile/internals.go | 10 +- daemon/commit.go | 104 +++- daemon/container.go | 11 +- daemon/container_unit_test.go | 2 +- daemon/container_unix.go | 11 +- daemon/container_windows.go | 26 +- daemon/create.go | 21 +- daemon/daemon.go | 495 ++++++++++++------ daemon/daemon_unix.go | 12 +- daemon/daemon_windows.go | 79 ++- daemon/daemonbuilder/builder.go | 42 +- daemon/delete.go | 11 +- daemon/errors.go | 24 +- daemon/events/filter.go | 9 +- daemon/graphdriver/fsdiff.go | 10 +- daemon/graphdriver/imagerestorer.go | 31 -- daemon/graphdriver/plugin.go | 1 - daemon/graphdriver/proxy.go | 1 - daemon/graphdriver/vfs/driver.go | 9 +- daemon/graphdriver/vfs/driver_unsupported.go | 3 - daemon/graphdriver/windows/windows.go | 196 ++++--- daemon/image_delete.go | 157 +++--- daemon/images.go | 163 ++++++ daemon/import.go | 111 ++++ daemon/info.go | 2 +- daemon/inspect.go | 16 +- daemon/list.go | 54 +- errors/daemon.go | 9 - integration-cli/docker_api_containers_test.go | 4 +- integration-cli/docker_cli_build_test.go | 4 +- integration-cli/docker_cli_by_digest_test.go | 10 +- integration-cli/docker_cli_create_test.go | 36 ++ ...cker_cli_external_graphdriver_unix_test.go | 3 +- integration-cli/docker_cli_images_test.go | 10 +- integration-cli/docker_cli_inspect_test.go | 7 +- integration-cli/docker_cli_pull_local_test.go | 68 +++ integration-cli/docker_cli_pull_test.go | 271 +--------- integration-cli/docker_cli_push_test.go | 43 -- integration-cli/docker_cli_rmi_test.go | 3 +- integration-cli/docker_cli_run_test.go | 12 + integration-cli/docker_cli_save_load_test.go | 32 +- .../docker_cli_save_load_unix_test.go | 8 +- integration-cli/docker_cli_tag_test.go | 58 +- pkg/parsers/parsers.go | 20 - pkg/parsers/parsers_test.go | 30 -- pkg/stringid/stringid.go | 4 + registry/config.go | 163 +++--- registry/registry_mock_test.go | 31 +- registry/registry_test.go | 219 +++++--- registry/service.go | 56 +- registry/service_v1.go | 12 +- registry/service_v2.go | 12 +- registry/session.go | 29 +- registry/types.go | 10 +- utils/utils.go | 17 - utils/utils_test.go | 30 -- 70 files changed, 2026 insertions(+), 1271 deletions(-) delete mode 100644 daemon/graphdriver/imagerestorer.go delete mode 100644 daemon/graphdriver/vfs/driver_unsupported.go create mode 100644 daemon/images.go create mode 100644 daemon/import.go diff --git a/api/client/build.go b/api/client/build.go index d4e2d514e4fe5..9b4829218779f 100644 --- a/api/client/build.go +++ b/api/client/build.go @@ -18,16 +18,15 @@ import ( "strconv" "strings" + "github.com/docker/distribution/reference" "github.com/docker/docker/api" Cli "github.com/docker/docker/cli" - "github.com/docker/docker/graph/tags" "github.com/docker/docker/opts" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/jsonmessage" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/progressreader" "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/ulimit" @@ -35,6 +34,7 @@ import ( "github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" + tagpkg "github.com/docker/docker/tag" "github.com/docker/docker/utils" ) @@ -323,7 +323,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { // Since the build was successful, now we must tag any of the resolved // images from the above Dockerfile rewrite. for _, resolved := range resolvedTags { - if err := cli.tagTrusted(resolved.repoInfo, resolved.digestRef, resolved.tagRef); err != nil { + if err := cli.tagTrusted(resolved.digestRef, resolved.tagRef); err != nil { return err } } @@ -333,16 +333,12 @@ func (cli *DockerCli) CmdBuild(args ...string) error { // validateTag checks if the given image name can be resolved. func validateTag(rawRepo string) (string, error) { - repository, tag := parsers.ParseRepositoryTag(rawRepo) - if err := registry.ValidateRepositoryName(repository); err != nil { + ref, err := reference.ParseNamed(rawRepo) + if err != nil { return "", err } - if len(tag) == 0 { - return rawRepo, nil - } - - if err := tags.ValidateTagName(tag); err != nil { + if err := registry.ValidateRepositoryName(ref); err != nil { return "", err } @@ -565,15 +561,16 @@ func (td *trustedDockerfile) Close() error { // resolvedTag records the repository, tag, and resolved digest reference // from a Dockerfile rewrite. type resolvedTag struct { - repoInfo *registry.RepositoryInfo - digestRef, tagRef registry.Reference + repoInfo *registry.RepositoryInfo + digestRef reference.Canonical + tagRef reference.NamedTagged } // rewriteDockerfileFrom rewrites the given Dockerfile by resolving images in // "FROM " instructions to a digest reference. `translator` is a // function that takes a repository name and tag reference and returns a // trusted digest reference. -func rewriteDockerfileFrom(dockerfileName string, translator func(string, registry.Reference) (registry.Reference, error)) (newDockerfile *trustedDockerfile, resolvedTags []*resolvedTag, err error) { +func rewriteDockerfileFrom(dockerfileName string, translator func(reference.NamedTagged) (reference.Canonical, error)) (newDockerfile *trustedDockerfile, resolvedTags []*resolvedTag, err error) { dockerfile, err := os.Open(dockerfileName) if err != nil { return nil, nil, fmt.Errorf("unable to open Dockerfile: %v", err) @@ -607,29 +604,39 @@ func rewriteDockerfileFrom(dockerfileName string, translator func(string, regist matches := dockerfileFromLinePattern.FindStringSubmatch(line) if matches != nil && matches[1] != "scratch" { // Replace the line with a resolved "FROM repo@digest" - repo, tag := parsers.ParseRepositoryTag(matches[1]) - if tag == "" { - tag = tags.DefaultTag + ref, err := reference.ParseNamed(matches[1]) + if err != nil { + return nil, nil, err } - repoInfo, err := registry.ParseRepositoryInfo(repo) - if err != nil { - return nil, nil, fmt.Errorf("unable to parse repository info %q: %v", repo, err) + digested := false + switch ref.(type) { + case reference.Tagged: + case reference.Digested: + digested = true + default: + ref, err = reference.WithTag(ref, tagpkg.DefaultTag) + if err != nil { + return nil, nil, err + } } - ref := registry.ParseReference(tag) + repoInfo, err := registry.ParseRepositoryInfo(ref) + if err != nil { + return nil, nil, fmt.Errorf("unable to parse repository info %q: %v", ref.String(), err) + } - if !ref.HasDigest() && isTrusted() { - trustedRef, err := translator(repo, ref) + if !digested && isTrusted() { + trustedRef, err := translator(ref.(reference.NamedTagged)) if err != nil { return nil, nil, err } - line = dockerfileFromLinePattern.ReplaceAllLiteralString(line, fmt.Sprintf("FROM %s", trustedRef.ImageName(repo))) + line = dockerfileFromLinePattern.ReplaceAllLiteralString(line, fmt.Sprintf("FROM %s", trustedRef.String())) resolvedTags = append(resolvedTags, &resolvedTag{ repoInfo: repoInfo, digestRef: trustedRef, - tagRef: ref, + tagRef: ref.(reference.NamedTagged), }) } } diff --git a/api/client/commit.go b/api/client/commit.go index 451813e3764cd..36fcb3c77e242 100644 --- a/api/client/commit.go +++ b/api/client/commit.go @@ -2,14 +2,15 @@ package client import ( "encoding/json" + "errors" "fmt" "net/url" + "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" Cli "github.com/docker/docker/cli" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" ) @@ -32,20 +33,35 @@ func (cli *DockerCli) CmdCommit(args ...string) error { cmd.ParseFlags(args, true) var ( - name = cmd.Arg(0) - repository, tag = parsers.ParseRepositoryTag(cmd.Arg(1)) + name = cmd.Arg(0) + repositoryAndTag = cmd.Arg(1) + repositoryName string + tag string ) //Check if the given image name can be resolved - if repository != "" { - if err := registry.ValidateRepositoryName(repository); err != nil { + if repositoryAndTag != "" { + ref, err := reference.ParseNamed(repositoryAndTag) + if err != nil { return err } + if err := registry.ValidateRepositoryName(ref); err != nil { + return err + } + + repositoryName = ref.Name() + + switch x := ref.(type) { + case reference.Digested: + return errors.New("cannot commit to digest reference") + case reference.Tagged: + tag = x.Tag() + } } v := url.Values{} v.Set("container", name) - v.Set("repo", repository) + v.Set("repo", repositoryName) v.Set("tag", tag) v.Set("comment", *flComment) v.Set("author", *flAuthor) diff --git a/api/client/create.go b/api/client/create.go index 9ef0edab6c4ad..9443799e5523f 100644 --- a/api/client/create.go +++ b/api/client/create.go @@ -9,12 +9,12 @@ import ( "os" "strings" + "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" Cli "github.com/docker/docker/cli" - "github.com/docker/docker/graph/tags" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" + tagpkg "github.com/docker/docker/tag" ) func (cli *DockerCli) pullImage(image string) error { @@ -23,16 +23,28 @@ func (cli *DockerCli) pullImage(image string) error { func (cli *DockerCli) pullImageCustomOut(image string, out io.Writer) error { v := url.Values{} - repos, tag := parsers.ParseRepositoryTag(image) - // pull only the image tagged 'latest' if no tag was specified - if tag == "" { - tag = tags.DefaultTag + + ref, err := reference.ParseNamed(image) + if err != nil { + return err + } + + var tag string + switch x := ref.(type) { + case reference.Digested: + tag = x.Digest().String() + case reference.Tagged: + tag = x.Tag() + default: + // pull only the image tagged 'latest' if no tag was specified + tag = tagpkg.DefaultTag } - v.Set("fromImage", repos) + + v.Set("fromImage", ref.Name()) v.Set("tag", tag) // Resolve the Repository name from fqn to RepositoryInfo - repoInfo, err := registry.ParseRepositoryInfo(repos) + repoInfo, err := registry.ParseRepositoryInfo(ref) if err != nil { return err } @@ -94,39 +106,46 @@ func (cli *DockerCli) createContainer(config *runconfig.Config, hostConfig *runc defer containerIDFile.Close() } - repo, tag := parsers.ParseRepositoryTag(config.Image) - if tag == "" { - tag = tags.DefaultTag + ref, err := reference.ParseNamed(config.Image) + if err != nil { + return nil, err } - ref := registry.ParseReference(tag) - var trustedRef registry.Reference + isDigested := false + switch ref.(type) { + case reference.Tagged: + case reference.Digested: + isDigested = true + default: + ref, err = reference.WithTag(ref, tagpkg.DefaultTag) + if err != nil { + return nil, err + } + } - if isTrusted() && !ref.HasDigest() { + var trustedRef reference.Canonical + + if isTrusted() && !isDigested { var err error - trustedRef, err = cli.trustedReference(repo, ref) + trustedRef, err = cli.trustedReference(ref.(reference.NamedTagged)) if err != nil { return nil, err } - config.Image = trustedRef.ImageName(repo) + config.Image = trustedRef.String() } //create the container serverResp, err := cli.call("POST", "/containers/create?"+containerValues.Encode(), mergedConfig, nil) //if image not found try to pull it if serverResp.statusCode == 404 && strings.Contains(err.Error(), config.Image) { - fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", ref.ImageName(repo)) + fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", ref.String()) // we don't want to write to stdout anything apart from container.ID if err = cli.pullImageCustomOut(config.Image, cli.err); err != nil { return nil, err } - if trustedRef != nil && !ref.HasDigest() { - repoInfo, err := registry.ParseRepositoryInfo(repo) - if err != nil { - return nil, err - } - if err := cli.tagTrusted(repoInfo, trustedRef, ref); err != nil { + if trustedRef != nil && !isDigested { + if err := cli.tagTrusted(trustedRef, ref.(reference.NamedTagged)); err != nil { return nil, err } } diff --git a/api/client/images.go b/api/client/images.go index d47deb62d399c..ba26512a2d31c 100644 --- a/api/client/images.go +++ b/api/client/images.go @@ -4,18 +4,18 @@ import ( "encoding/json" "fmt" "net/url" + "strings" "text/tabwriter" "time" + "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" Cli "github.com/docker/docker/cli" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/parsers/filters" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/units" - "github.com/docker/docker/utils" ) // CmdImages lists the images in a specified repository, or all top-level images if no repository is specified. @@ -78,9 +78,9 @@ func (cli *DockerCli) CmdImages(args ...string) error { w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) if !*quiet { if *showDigests { - fmt.Fprintln(w, "REPOSITORY\tTAG\tDIGEST\tIMAGE ID\tCREATED\tVIRTUAL SIZE") + fmt.Fprintln(w, "REPOSITORY\tTAG\tDIGEST\tIMAGE ID\tCREATED\tSIZE") } else { - fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tVIRTUAL SIZE") + fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tSIZE") } } @@ -101,21 +101,31 @@ func (cli *DockerCli) CmdImages(args ...string) error { // combine the tags and digests lists tagsAndDigests := append(repoTags, repoDigests...) for _, repoAndRef := range tagsAndDigests { - repo, ref := parsers.ParseRepositoryTag(repoAndRef) - // default tag and digest to none - if there's a value, it'll be set below + // default repo, tag, and digest to none - if there's a value, it'll be set below + repo := "" tag := "" digest := "" - if utils.DigestReference(ref) { - digest = ref - } else { - tag = ref + + if !strings.HasPrefix(repoAndRef, "") { + ref, err := reference.ParseNamed(repoAndRef) + if err != nil { + return err + } + repo = ref.Name() + + switch x := ref.(type) { + case reference.Digested: + digest = x.Digest().String() + case reference.Tagged: + tag = x.Tag() + } } if !*quiet { if *showDigests { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, tag, digest, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.VirtualSize))) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, tag, digest, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.Size))) } else { - fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.VirtualSize))) + fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, ID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(int64(image.Created), 0))), units.HumanSize(float64(image.Size))) } } else { fmt.Fprintln(w, ID) diff --git a/api/client/import.go b/api/client/import.go index c64e88d3c19d0..2838debd98a97 100644 --- a/api/client/import.go +++ b/api/client/import.go @@ -6,10 +6,10 @@ import ( "net/url" "os" + "github.com/docker/distribution/reference" Cli "github.com/docker/docker/cli" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/registry" ) @@ -47,8 +47,11 @@ func (cli *DockerCli) CmdImport(args ...string) error { if repository != "" { //Check if the given image name can be resolved - repo, _ := parsers.ParseRepositoryTag(repository) - if err := registry.ValidateRepositoryName(repo); err != nil { + ref, err := reference.ParseNamed(repository) + if err != nil { + return err + } + if err := registry.ValidateRepositoryName(ref); err != nil { return err } } diff --git a/api/client/ps/custom.go b/api/client/ps/custom.go index 1739fd12a61d7..77e460892be63 100644 --- a/api/client/ps/custom.go +++ b/api/client/ps/custom.go @@ -62,8 +62,8 @@ func (c *containerContext) Image() string { return "" } if c.trunc { - if stringid.TruncateID(c.c.ImageID) == stringid.TruncateID(c.c.Image) { - return stringutils.Truncate(c.c.Image, 12) + if trunc := stringid.TruncateID(c.c.ImageID); trunc == stringid.TruncateID(c.c.Image) { + return trunc } } return c.c.Image diff --git a/api/client/pull.go b/api/client/pull.go index dcb2ecb0c70f7..e585a12081a5f 100644 --- a/api/client/pull.go +++ b/api/client/pull.go @@ -1,16 +1,19 @@ package client import ( + "errors" "fmt" "net/url" + "github.com/docker/distribution/reference" Cli "github.com/docker/docker/cli" - "github.com/docker/docker/graph/tags" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" + tagpkg "github.com/docker/docker/tag" ) +var errTagCantBeUsed = errors.New("tag can't be used with --all-tags/-a") + // CmdPull pulls an image or a repository from the registry. // // Usage: docker pull [OPTIONS] IMAGENAME[:TAG|@DIGEST] @@ -23,18 +26,38 @@ func (cli *DockerCli) CmdPull(args ...string) error { cmd.ParseFlags(args, true) remote := cmd.Arg(0) - taglessRemote, tag := parsers.ParseRepositoryTag(remote) - if tag == "" && !*allTags { - tag = tags.DefaultTag - fmt.Fprintf(cli.out, "Using default tag: %s\n", tag) - } else if tag != "" && *allTags { - return fmt.Errorf("tag can't be used with --all-tags/-a") + distributionRef, err := reference.ParseNamed(remote) + if err != nil { + return err + } + + var tag string + switch x := distributionRef.(type) { + case reference.Digested: + if *allTags { + return errTagCantBeUsed + } + tag = x.Digest().String() + case reference.Tagged: + if *allTags { + return errTagCantBeUsed + } + tag = x.Tag() + default: + if !*allTags { + tag = tagpkg.DefaultTag + distributionRef, err = reference.WithTag(distributionRef, tag) + if err != nil { + return err + } + fmt.Fprintf(cli.out, "Using default tag: %s\n", tag) + } } ref := registry.ParseReference(tag) // Resolve the Repository name from fqn to RepositoryInfo - repoInfo, err := registry.ParseRepositoryInfo(taglessRemote) + repoInfo, err := registry.ParseRepositoryInfo(distributionRef) if err != nil { return err } @@ -46,7 +69,7 @@ func (cli *DockerCli) CmdPull(args ...string) error { } v := url.Values{} - v.Set("fromImage", ref.ImageName(taglessRemote)) + v.Set("fromImage", distributionRef.String()) _, _, err = cli.clientRequestAttemptLogin("POST", "/images/create?"+v.Encode(), nil, cli.out, repoInfo.Index, "pull") return err diff --git a/api/client/push.go b/api/client/push.go index 0c8823f051034..760c97ef8bde6 100644 --- a/api/client/push.go +++ b/api/client/push.go @@ -1,12 +1,13 @@ package client import ( + "errors" "fmt" "net/url" + "github.com/docker/distribution/reference" Cli "github.com/docker/docker/cli" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" ) @@ -20,10 +21,21 @@ func (cli *DockerCli) CmdPush(args ...string) error { cmd.ParseFlags(args, true) - remote, tag := parsers.ParseRepositoryTag(cmd.Arg(0)) + ref, err := reference.ParseNamed(cmd.Arg(0)) + if err != nil { + return err + } + + var tag string + switch x := ref.(type) { + case reference.Digested: + return errors.New("cannot push a digest reference") + case reference.Tagged: + tag = x.Tag() + } // Resolve the Repository name from fqn to RepositoryInfo - repoInfo, err := registry.ParseRepositoryInfo(remote) + repoInfo, err := registry.ParseRepositoryInfo(ref) if err != nil { return err } @@ -48,6 +60,6 @@ func (cli *DockerCli) CmdPush(args ...string) error { v := url.Values{} v.Set("tag", tag) - _, _, err = cli.clientRequestAttemptLogin("POST", "/images/"+remote+"/push?"+v.Encode(), nil, cli.out, repoInfo.Index, "push") + _, _, err = cli.clientRequestAttemptLogin("POST", "/images/"+ref.Name()+"/push?"+v.Encode(), nil, cli.out, repoInfo.Index, "push") return err } diff --git a/api/client/search.go b/api/client/search.go index f02d5fe8b0678..1a47064477bd7 100644 --- a/api/client/search.go +++ b/api/client/search.go @@ -10,7 +10,6 @@ import ( Cli "github.com/docker/docker/cli" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/stringutils" "github.com/docker/docker/registry" ) @@ -38,10 +37,7 @@ func (cli *DockerCli) CmdSearch(args ...string) error { v := url.Values{} v.Set("term", name) - // Resolve the Repository name from fqn to hostname + name - taglessRemote, _ := parsers.ParseRepositoryTag(name) - - indexInfo, err := registry.ParseIndexInfo(taglessRemote) + indexInfo, err := registry.ParseSearchIndexInfo(name) if err != nil { return err } diff --git a/api/client/tag.go b/api/client/tag.go index 505e0208cda89..99a5a43bbf532 100644 --- a/api/client/tag.go +++ b/api/client/tag.go @@ -1,11 +1,12 @@ package client import ( + "errors" "net/url" + "github.com/docker/distribution/reference" Cli "github.com/docker/docker/cli" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" ) @@ -19,16 +20,28 @@ func (cli *DockerCli) CmdTag(args ...string) error { cmd.ParseFlags(args, true) - var ( - repository, tag = parsers.ParseRepositoryTag(cmd.Arg(1)) - v = url.Values{} - ) + v := url.Values{} + ref, err := reference.ParseNamed(cmd.Arg(1)) + if err != nil { + return err + } + + _, isDigested := ref.(reference.Digested) + if isDigested { + return errors.New("refusing to create a tag with a digest reference") + } + + tag := "" + tagged, isTagged := ref.(reference.Tagged) + if isTagged { + tag = tagged.Tag() + } //Check if the given image name can be resolved - if err := registry.ValidateRepositoryName(repository); err != nil { + if err := registry.ValidateRepositoryName(ref); err != nil { return err } - v.Set("repo", repository) + v.Set("repo", ref.Name()) v.Set("tag", tag) if *force { diff --git a/api/client/trust.go b/api/client/trust.go index 00dbf6de6223f..da6c3e5768b55 100644 --- a/api/client/trust.go +++ b/api/client/trust.go @@ -19,6 +19,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/digest" + "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/cliconfig" @@ -163,12 +164,12 @@ func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, aut } creds := simpleCredentialStore{auth: authConfig} - tokenHandler := auth.NewTokenHandler(authTransport, creds, repoInfo.CanonicalName, "push", "pull") + tokenHandler := auth.NewTokenHandler(authTransport, creds, repoInfo.CanonicalName.Name(), "push", "pull") basicHandler := auth.NewBasicHandler(creds) modifiers = append(modifiers, transport.RequestModifier(auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))) tr := transport.NewTransport(base, modifiers...) - return client.NewNotaryRepository(cli.trustDirectory(), repoInfo.CanonicalName, server, tr, cli.getPassphraseRetriever()) + return client.NewNotaryRepository(cli.trustDirectory(), repoInfo.CanonicalName.Name(), server, tr, cli.getPassphraseRetriever()) } func convertTarget(t client.Target) (target, error) { @@ -219,8 +220,8 @@ func (cli *DockerCli) getPassphraseRetriever() passphrase.Retriever { } } -func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (registry.Reference, error) { - repoInfo, err := registry.ParseRepositoryInfo(repo) +func (cli *DockerCli) trustedReference(ref reference.NamedTagged) (reference.Canonical, error) { + repoInfo, err := registry.ParseRepositoryInfo(ref) if err != nil { return nil, err } @@ -234,7 +235,7 @@ func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (reg return nil, err } - t, err := notaryRepo.GetTargetByName(ref.String()) + t, err := notaryRepo.GetTargetByName(ref.Tag()) if err != nil { return nil, err } @@ -244,18 +245,17 @@ func (cli *DockerCli) trustedReference(repo string, ref registry.Reference) (reg } - return registry.DigestReference(r.digest), nil + return reference.WithDigest(ref, r.digest) } -func (cli *DockerCli) tagTrusted(repoInfo *registry.RepositoryInfo, trustedRef, ref registry.Reference) error { - fullName := trustedRef.ImageName(repoInfo.LocalName) - fmt.Fprintf(cli.out, "Tagging %s as %s\n", fullName, ref.ImageName(repoInfo.LocalName)) +func (cli *DockerCli) tagTrusted(trustedRef reference.Canonical, ref reference.NamedTagged) error { + fmt.Fprintf(cli.out, "Tagging %s as %s\n", trustedRef.String(), ref.String()) tv := url.Values{} - tv.Set("repo", repoInfo.LocalName) - tv.Set("tag", ref.String()) + tv.Set("repo", trustedRef.Name()) + tv.Set("tag", ref.Tag()) tv.Set("force", "1") - if _, _, err := readBody(cli.call("POST", "/images/"+fullName+"/tag?"+tv.Encode(), nil, nil)); err != nil { + if _, _, err := readBody(cli.call("POST", "/images/"+trustedRef.String()+"/tag?"+tv.Encode(), nil, nil)); err != nil { return err } @@ -317,7 +317,7 @@ func (cli *DockerCli) trustedPull(repoInfo *registry.RepositoryInfo, ref registr refs = append(refs, r) } - v.Set("fromImage", repoInfo.LocalName) + v.Set("fromImage", repoInfo.LocalName.Name()) for i, r := range refs { displayTag := r.reference.String() if displayTag != "" { @@ -333,7 +333,12 @@ func (cli *DockerCli) trustedPull(repoInfo *registry.RepositoryInfo, ref registr // If reference is not trusted, tag by trusted reference if !r.reference.HasDigest() { - if err := cli.tagTrusted(repoInfo, registry.DigestReference(r.digest), r.reference); err != nil { + tagged, err := reference.WithTag(repoInfo.LocalName, r.reference.String()) + if err != nil { + return err + } + trustedRef, err := reference.WithDigest(repoInfo.LocalName, r.digest) + if err := cli.tagTrusted(trustedRef, tagged); err != nil { return err } @@ -386,7 +391,7 @@ func (cli *DockerCli) trustedPush(repoInfo *registry.RepositoryInfo, tag string, v := url.Values{} v.Set("tag", tag) - _, _, err := cli.clientRequestAttemptLogin("POST", "/images/"+repoInfo.LocalName+"/push?"+v.Encode(), nil, streamOut, repoInfo.Index, "push") + _, _, err := cli.clientRequestAttemptLogin("POST", "/images/"+repoInfo.LocalName.Name()+"/push?"+v.Encode(), nil, streamOut, repoInfo.Index, "push") // Close stream channel to finish target parsing if err := streamOut.Close(); err != nil { return err diff --git a/api/server/router/local/image.go b/api/server/router/local/image.go index 7e41eac894757..77f58f6b66dba 100644 --- a/api/server/router/local/image.go +++ b/api/server/router/local/image.go @@ -10,6 +10,8 @@ import ( "strings" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/digest" + "github.com/docker/distribution/reference" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" "github.com/docker/docker/builder" @@ -17,17 +19,14 @@ import ( "github.com/docker/docker/cliconfig" "github.com/docker/docker/daemon/daemonbuilder" derr "github.com/docker/docker/errors" - "github.com/docker/docker/graph" - "github.com/docker/docker/graph/tags" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/progressreader" "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/ulimit" - "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" + tagpkg "github.com/docker/docker/tag" "github.com/docker/docker/utils" "golang.org/x/net/context" ) @@ -110,26 +109,55 @@ func (s *router) postImagesCreate(ctx context.Context, w http.ResponseWriter, r w.Header().Set("Content-Type", "application/json") if image != "" { //pull - if tag == "" { - image, tag = parsers.ParseRepositoryTag(image) - } - metaHeaders := map[string][]string{} - for k, v := range r.Header { - if strings.HasPrefix(k, "X-Meta-") { - metaHeaders[k] = v + // Special case: "pull -a" may send an image name with a + // trailing :. This is ugly, but let's not break API + // compatibility. + image = strings.TrimSuffix(image, ":") + + var ref reference.Named + ref, err = reference.ParseNamed(image) + if err == nil { + if tag != "" { + // The "tag" could actually be a digest. + var dgst digest.Digest + dgst, err = digest.ParseDigest(tag) + if err == nil { + ref, err = reference.WithDigest(ref, dgst) + } else { + ref, err = reference.WithTag(ref, tag) + } + } + if err == nil { + metaHeaders := map[string][]string{} + for k, v := range r.Header { + if strings.HasPrefix(k, "X-Meta-") { + metaHeaders[k] = v + } + } + + err = s.daemon.PullImage(ref, metaHeaders, authConfig, output) } } + } else { //import + var newRef reference.Named + if repo != "" { + var err error + newRef, err = reference.ParseNamed(repo) + if err != nil { + return err + } - imagePullConfig := &graph.ImagePullConfig{ - MetaHeaders: metaHeaders, - AuthConfig: authConfig, - OutStream: output, - } + switch newRef.(type) { + case reference.Digested: + return errors.New("cannot import digest reference") + } - err = s.daemon.PullImage(image, tag, imagePullConfig) - } else { //import - if tag == "" { - repo, tag = parsers.ParseRepositoryTag(repo) + if tag != "" { + newRef, err = reference.WithTag(newRef, tag) + if err != nil { + return err + } + } } src := r.Form.Get("fromSrc") @@ -143,7 +171,7 @@ func (s *router) postImagesCreate(ctx context.Context, w http.ResponseWriter, r return err } - err = s.daemon.ImportImage(src, repo, tag, message, r.Body, output, newConfig) + err = s.daemon.ImportImage(src, newRef, message, r.Body, output, newConfig) } if err != nil { if !output.Flushed() { @@ -183,19 +211,25 @@ func (s *router) postImagesPush(ctx context.Context, w http.ResponseWriter, r *h } } - name := vars["name"] + ref, err := reference.ParseNamed(vars["name"]) + if err != nil { + return err + } + tag := r.Form.Get("tag") + if tag != "" { + // Push by digest is not supported, so only tags are supported. + ref, err = reference.WithTag(ref, tag) + if err != nil { + return err + } + } + output := ioutils.NewWriteFlusher(w) defer output.Close() - imagePushConfig := &graph.ImagePushConfig{ - MetaHeaders: metaHeaders, - AuthConfig: authConfig, - Tag: r.Form.Get("tag"), - OutStream: output, - } w.Header().Set("Content-Type", "application/json") - if err := s.daemon.PushImage(name, imagePushConfig); err != nil { + if err := s.daemon.PushImage(ref, metaHeaders, authConfig, output); err != nil { if !output.Flushed() { return err } @@ -428,7 +462,7 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R } for _, rt := range repoAndTags { - if err := s.daemon.TagImage(rt.repo, rt.tag, string(imgID), true); err != nil { + if err := s.daemon.TagImage(rt, imgID, true); err != nil { return errf(err) } } @@ -436,43 +470,38 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R return nil } -// repoAndTag is a helper struct for holding the parsed repositories and tags of -// the input "t" argument. -type repoAndTag struct { - repo, tag string -} - // sanitizeRepoAndTags parses the raw "t" parameter received from the client // to a slice of repoAndTag. // It also validates each repoName and tag. -func sanitizeRepoAndTags(names []string) ([]repoAndTag, error) { +func sanitizeRepoAndTags(names []string) ([]reference.Named, error) { var ( - repoAndTags []repoAndTag + repoAndTags []reference.Named // This map is used for deduplicating the "-t" paramter. uniqNames = make(map[string]struct{}) ) for _, repo := range names { - name, tag := parsers.ParseRepositoryTag(repo) - if name == "" { + if repo == "" { continue } - if err := registry.ValidateRepositoryName(name); err != nil { + ref, err := reference.ParseNamed(repo) + if err != nil { return nil, err } - nameWithTag := name - if len(tag) > 0 { - if err := tags.ValidateTagName(tag); err != nil { - return nil, err - } - nameWithTag += ":" + tag - } else { - nameWithTag += ":" + tags.DefaultTag + if _, isDigested := ref.(reference.Digested); isDigested { + return nil, errors.New("build tag cannot be a digest") + } + + if _, isTagged := ref.(reference.Tagged); !isTagged { + ref, err = reference.WithTag(ref, tagpkg.DefaultTag) } + + nameWithTag := ref.String() + if _, exists := uniqNames[nameWithTag]; !exists { uniqNames[nameWithTag] = struct{}{} - repoAndTags = append(repoAndTags, repoAndTag{repo: name, tag: tag}) + repoAndTags = append(repoAndTags, ref) } } return repoAndTags, nil @@ -484,7 +513,7 @@ func (s *router) getImagesJSON(ctx context.Context, w http.ResponseWriter, r *ht } // FIXME: The filter parameter could just be a match filter - images, err := s.daemon.ListImages(r.Form.Get("filters"), r.Form.Get("filter"), httputils.BoolValue(r, "all")) + images, err := s.daemon.Images(r.Form.Get("filters"), r.Form.Get("filter"), httputils.BoolValue(r, "all")) if err != nil { return err } @@ -508,9 +537,17 @@ func (s *router) postImagesTag(ctx context.Context, w http.ResponseWriter, r *ht } repo := r.Form.Get("repo") tag := r.Form.Get("tag") - name := vars["name"] + newTag, err := reference.WithName(repo) + if err != nil { + return err + } + if tag != "" { + if newTag, err = reference.WithTag(newTag, tag); err != nil { + return err + } + } force := httputils.BoolValue(r, "force") - if err := s.daemon.TagImage(repo, tag, name, force); err != nil { + if err := s.daemon.TagImage(newTag, vars["name"], force); err != nil { return err } w.WriteHeader(http.StatusCreated) diff --git a/builder/builder.go b/builder/builder.go index ac045fe41e6cf..e5d4d63f9bf10 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -125,7 +125,7 @@ type Docker interface { // Remove removes a container specified by `id`. Remove(id string, cfg *daemon.ContainerRmConfig) error // Commit creates a new Docker image from an existing Docker container. - Commit(string, *daemon.ContainerCommitConfig) (*image.Image, error) + Commit(string, *daemon.ContainerCommitConfig) (string, error) // Copy copies/extracts a source FileInfo to a destination path inside a container // specified by a container object. // TODO: make an Extract method instead of passing `decompress` diff --git a/builder/dockerfile/builder.go b/builder/dockerfile/builder.go index bd4283daf9fcf..77820a9715f9a 100644 --- a/builder/dockerfile/builder.go +++ b/builder/dockerfile/builder.go @@ -277,9 +277,9 @@ func Commit(containerName string, d *daemon.Daemon, c *CommitConfig) (string, er MergeConfigs: true, } - img, err := d.Commit(containerName, commitCfg) + imgID, err := d.Commit(containerName, commitCfg) if err != nil { return "", err } - return img.ID, nil + return imgID, nil } diff --git a/builder/dockerfile/internals.go b/builder/dockerfile/internals.go index a251af4f6e8cb..cdb86cdd17a46 100644 --- a/builder/dockerfile/internals.go +++ b/builder/dockerfile/internals.go @@ -83,13 +83,13 @@ func (b *Builder) commit(id string, autoCmd *stringutils.StrSlice, comment strin } // Commit the container - image, err := b.docker.Commit(id, commitCfg) + imageID, err := b.docker.Commit(id, commitCfg) if err != nil { return err } - b.docker.Retain(b.id, image.ID) - b.activeImages = append(b.activeImages, image.ID) - b.image = image.ID + b.docker.Retain(b.id, imageID) + b.activeImages = append(b.activeImages, imageID) + b.image = imageID return nil } @@ -412,7 +412,7 @@ func containsWildcards(name string) bool { } func (b *Builder) processImageFrom(img *image.Image) error { - b.image = img.ID + b.image = img.ID().String() if img.Config != nil { b.runConfig = img.Config diff --git a/daemon/commit.go b/daemon/commit.go index 61808eabce5db..01bfa3c7ae24e 100644 --- a/daemon/commit.go +++ b/daemon/commit.go @@ -1,10 +1,16 @@ package daemon import ( + "encoding/json" "fmt" "runtime" + "strings" + "time" + "github.com/docker/distribution/reference" + "github.com/docker/docker/dockerversion" "github.com/docker/docker/image" + "github.com/docker/docker/layer" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/runconfig" @@ -25,15 +31,15 @@ type ContainerCommitConfig struct { // Commit creates a new filesystem image from the current state of a container. // The image can optionally be tagged into a repository. -func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (*image.Image, error) { +func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (string, error) { container, err := daemon.Get(name) if err != nil { - return nil, err + return "", err } // It is not possible to commit a running container on Windows if runtime.GOOS == "windows" && container.IsRunning() { - return nil, fmt.Errorf("Windows does not support commit of a running container") + return "", fmt.Errorf("Windows does not support commit of a running container") } if c.Pause && !container.isPaused() { @@ -43,13 +49,13 @@ func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (*image.Imag if c.MergeConfigs { if err := runconfig.Merge(c.Config, container.Config); err != nil { - return nil, err + return "", err } } rwTar, err := daemon.exportContainerRw(container) if err != nil { - return nil, err + return "", err } defer func() { if rwTar != nil { @@ -57,31 +63,99 @@ func (daemon *Daemon) Commit(name string, c *ContainerCommitConfig) (*image.Imag } }() - // Create a new image from the container's base layers + a new layer from container changes - img, err := daemon.graph.Create(rwTar, container.ID, container.ImageID, c.Comment, c.Author, container.Config, c.Config) + var history []image.History + rootFS := image.NewRootFS() + + if container.ImageID != "" { + img, err := daemon.imageStore.Get(container.ImageID) + if err != nil { + return "", err + } + history = img.History + rootFS = img.RootFS + } + + l, err := daemon.layerStore.Register(rwTar, rootFS.ChainID()) if err != nil { - return nil, err + return "", err + } + defer layer.ReleaseAndLog(daemon.layerStore, l) + + h := image.History{ + Author: c.Author, + Created: time.Now().UTC(), + CreatedBy: strings.Join(container.Config.Cmd.Slice(), " "), + Comment: c.Comment, + EmptyLayer: true, + } + + if diffID := l.DiffID(); layer.DigestSHA256EmptyTar != diffID { + h.EmptyLayer = false + rootFS.Append(diffID) + } + + history = append(history, h) + + config, err := json.Marshal(&image.Image{ + V1Image: image.V1Image{ + DockerVersion: dockerversion.Version, + Config: c.Config, + Architecture: runtime.GOARCH, + OS: runtime.GOOS, + Container: container.ID, + ContainerConfig: *container.Config, + Author: c.Author, + Created: h.Created, + }, + RootFS: rootFS, + History: history, + }) + + if err != nil { + return "", err + } + + id, err := daemon.imageStore.Create(config) + if err != nil { + return "", err + } + + if container.ImageID != "" { + if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil { + return "", err + } } - // Register the image if needed if c.Repo != "" { - if err := daemon.repositories.Tag(c.Repo, c.Tag, img.ID, true); err != nil { - return img, err + newTag, err := reference.WithName(c.Repo) // todo: should move this to API layer + if err != nil { + return "", err + } + if c.Tag != "" { + if newTag, err = reference.WithTag(newTag, c.Tag); err != nil { + return "", err + } + } + if err := daemon.TagImage(newTag, id.String(), true); err != nil { + return "", err } } daemon.LogContainerEvent(container, "commit") - return img, nil + return id.String(), nil } func (daemon *Daemon) exportContainerRw(container *Container) (archive.Archive, error) { - archive, err := daemon.diff(container) + if err := daemon.Mount(container); err != nil { + return nil, err + } + + archive, err := container.rwlayer.TarStream() if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(archive, func() error { - err := archive.Close() - return err + return daemon.layerStore.Unmount(container.ID) }), nil } diff --git a/daemon/container.go b/daemon/container.go index b15f4eebbc11a..e39093bb50076 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -20,6 +20,8 @@ import ( "github.com/docker/docker/daemon/logger/jsonfilelog" "github.com/docker/docker/daemon/network" derr "github.com/docker/docker/errors" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" "github.com/docker/docker/pkg/nat" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/signal" @@ -29,6 +31,8 @@ import ( "github.com/docker/docker/volume" ) +const configFileName = "config.v2.json" + var ( // ErrRootFSReadOnly is returned when a container // rootfs is marked readonly. @@ -43,12 +47,13 @@ type CommonContainer struct { *State `json:"State"` // Needed for remote api version <= 1.11 root string // Path to the "home" of the container, including metadata. basefs string // Path to the graphdriver mountpoint + rwlayer layer.RWLayer ID string Created time.Time Path string Args []string Config *runconfig.Config - ImageID string `json:"Image"` + ImageID image.ID `json:"Image"` NetworkSettings *network.Settings LogPath string Name string @@ -256,7 +261,7 @@ func (container *Container) hostConfigPath() (string, error) { } func (container *Container) jsonPath() (string, error) { - return container.getRootResourcePath("config.json") + return container.getRootResourcePath(configFileName) } // This directory is only usable when the container is running @@ -301,7 +306,7 @@ func (container *Container) StartLogger(cfg runconfig.LogConfig) (logger.Logger, ContainerName: container.Name, ContainerEntrypoint: container.Path, ContainerArgs: container.Args, - ContainerImageID: container.ImageID, + ContainerImageID: container.ImageID.String(), ContainerImageName: container.Config.Image, ContainerCreated: container.Created, ContainerEnv: container.Config.Env, diff --git a/daemon/container_unit_test.go b/daemon/container_unit_test.go index f7951804a3fa2..c42e7751f3b1b 100644 --- a/daemon/container_unit_test.go +++ b/daemon/container_unit_test.go @@ -99,7 +99,7 @@ func TestContainerInitDNS(t *testing.T) { "Name":"/ubuntu","Driver":"aufs","MountLabel":"","ProcessLabel":"","AppArmorProfile":"","RestartCount":0, "UpdateDns":false,"Volumes":{},"VolumesRW":{},"AppliedVolumesFrom":null}` - if err = ioutil.WriteFile(filepath.Join(containerPath, "config.json"), []byte(config), 0644); err != nil { + if err = ioutil.WriteFile(filepath.Join(containerPath, configFileName), []byte(config), 0644); err != nil { t.Fatal(err) } diff --git a/daemon/container_unix.go b/daemon/container_unix.go index a925717fe61c1..6d6ee0e7ade07 100644 --- a/daemon/container_unix.go +++ b/daemon/container_unix.go @@ -19,7 +19,6 @@ import ( "github.com/docker/docker/daemon/links" "github.com/docker/docker/daemon/network" derr "github.com/docker/docker/errors" - "github.com/docker/docker/pkg/directory" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/mount" @@ -388,8 +387,7 @@ func (daemon *Daemon) getSize(container *Container) (int64, int64) { } defer daemon.Unmount(container) - initID := fmt.Sprintf("%s-init", container.ID) - sizeRw, err = daemon.driver.DiffSize(container.ID, initID) + sizeRw, err = container.rwlayer.Size() if err != nil { logrus.Errorf("Driver %s couldn't return diff size of container %s: %s", daemon.driver, container.ID, err) // FIXME: GetSize should return an error. Not changing it now in case @@ -397,9 +395,12 @@ func (daemon *Daemon) getSize(container *Container) (int64, int64) { sizeRw = -1 } - if _, err = os.Stat(container.basefs); err == nil { - if sizeRootfs, err = directory.Size(container.basefs); err != nil { + if parent := container.rwlayer.Parent(); parent != nil { + sizeRootfs, err = parent.Size() + if err != nil { sizeRootfs = -1 + } else if sizeRw != -1 { + sizeRootfs += sizeRw } } return sizeRw, sizeRootfs diff --git a/daemon/container_windows.go b/daemon/container_windows.go index 5400f12f45506..9563f0d26290b 100644 --- a/daemon/container_windows.go +++ b/daemon/container_windows.go @@ -7,6 +7,7 @@ import ( "github.com/docker/docker/daemon/execdriver" derr "github.com/docker/docker/errors" + "github.com/docker/docker/layer" "github.com/docker/docker/volume" "github.com/docker/libnetwork" ) @@ -98,22 +99,25 @@ func (daemon *Daemon) populateCommand(c *Container, env []string) error { processConfig.Env = env var layerPaths []string - img, err := daemon.graph.Get(c.ImageID) + img, err := daemon.imageStore.Get(c.ImageID) if err != nil { return derr.ErrorCodeGetGraph.WithArgs(c.ImageID, err) } - for i := img; i != nil && err == nil; i, err = daemon.graph.GetParent(i) { - lp, err := daemon.driver.Get(i.ID, "") - if err != nil { - return derr.ErrorCodeGetLayer.WithArgs(daemon.driver.String(), i.ID, err) - } - layerPaths = append(layerPaths, lp) - err = daemon.driver.Put(i.ID) - if err != nil { - return derr.ErrorCodePutLayer.WithArgs(daemon.driver.String(), i.ID, err) + + if img.RootFS != nil && img.RootFS.Type == "layers+base" { + max := len(img.RootFS.DiffIDs) + for i := 0; i <= max; i++ { + img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i] + path, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID()) + if err != nil { + return derr.ErrorCodeGetLayer.WithArgs(err) + } + // Reverse order, expecting parent most first + layerPaths = append([]string{path}, layerPaths...) } } - m, err := daemon.driver.GetMetadata(c.ID) + + m, err := layer.RWLayerMetadata(daemon.layerStore, c.ID) if err != nil { return derr.ErrorCodeGetLayerMetadata.WithArgs(err) } diff --git a/daemon/create.go b/daemon/create.go index 2c26931733b2b..3ae2e9cdeb8b7 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -5,6 +5,7 @@ import ( "github.com/docker/docker/api/types" derr "github.com/docker/docker/errors" "github.com/docker/docker/image" + "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/runconfig" "github.com/docker/docker/volume" @@ -34,7 +35,7 @@ func (daemon *Daemon) ContainerCreate(params *ContainerCreateConfig) (types.Cont container, err := daemon.create(params) if err != nil { - return types.ContainerCreateResponse{ID: "", Warnings: warnings}, daemon.graphNotExistToErrcode(params.Config.Image, err) + return types.ContainerCreateResponse{ID: "", Warnings: warnings}, daemon.imageNotExistToErrcode(err) } return types.ContainerCreateResponse{ID: container.ID, Warnings: warnings}, nil @@ -45,19 +46,16 @@ func (daemon *Daemon) create(params *ContainerCreateConfig) (retC *Container, re var ( container *Container img *image.Image - imgID string + imgID image.ID err error ) if params.Config.Image != "" { - img, err = daemon.repositories.LookupImage(params.Config.Image) + img, err = daemon.GetImage(params.Config.Image) if err != nil { return nil, err } - if err = daemon.graph.CheckDepth(img); err != nil { - return nil, err - } - imgID = img.ID + imgID = img.ID() } if err := daemon.mergeAndVerifyConfig(params.Config, img); err != nil { @@ -87,15 +85,14 @@ func (daemon *Daemon) create(params *ContainerCreateConfig) (retC *Container, re if err := daemon.Register(container); err != nil { return nil, err } - container.Lock() - if err := parseSecurityOpt(container, params.HostConfig); err != nil { - container.Unlock() + rootUID, rootGID, err := idtools.GetRootUIDGID(daemon.uidMaps, daemon.gidMaps) + if err != nil { return nil, err } - container.Unlock() - if err := daemon.createRootfs(container); err != nil { + if err := idtools.MkdirAs(container.root, 0700, rootUID, rootGID); err != nil { return nil, err } + if err := daemon.setHostConfig(container, params.HostConfig); err != nil { return nil, err } diff --git a/daemon/daemon.go b/daemon/daemon.go index 435c93db31bec..6a8d18e92302e 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -18,6 +18,8 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/digest" + "github.com/docker/distribution/reference" "github.com/docker/docker/api" "github.com/docker/docker/api/types" "github.com/docker/docker/cliconfig" @@ -29,9 +31,13 @@ import ( _ "github.com/docker/docker/daemon/graphdriver/vfs" // register vfs "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/network" + "github.com/docker/docker/distribution" + dmetadata "github.com/docker/docker/distribution/metadata" derr "github.com/docker/docker/errors" - "github.com/docker/docker/graph" "github.com/docker/docker/image" + "github.com/docker/docker/image/tarexport" + "github.com/docker/docker/layer" + "github.com/docker/docker/migrate/v1" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/discovery" "github.com/docker/docker/pkg/fileutils" @@ -50,12 +56,14 @@ import ( "github.com/docker/docker/pkg/truncindex" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" + "github.com/docker/docker/tag" "github.com/docker/docker/utils" volumedrivers "github.com/docker/docker/volume/drivers" "github.com/docker/docker/volume/local" "github.com/docker/docker/volume/store" "github.com/docker/libnetwork" lntypes "github.com/docker/libnetwork/types" + "github.com/docker/libtrust" "github.com/opencontainers/runc/libcontainer" ) @@ -66,6 +74,15 @@ var ( errSystemNotSupported = errors.New("The Docker daemon is not supported on this platform.") ) +// ErrImageDoesNotExist is error returned when no image can be found for a reference. +type ErrImageDoesNotExist struct { + RefOrID string +} + +func (e ErrImageDoesNotExist) Error() string { + return fmt.Sprintf("no such id: %s", e.RefOrID) +} + type contStore struct { s map[string]*Container sync.Mutex @@ -103,29 +120,33 @@ func (c *contStore) List() []*Container { // Daemon holds information about the Docker daemon. type Daemon struct { - ID string - repository string - sysInitPath string - containers *contStore - execCommands *exec.Store - graph *graph.Graph - repositories *graph.TagStore - idIndex *truncindex.TruncIndex - configStore *Config - containerGraphDB *graphdb.Database - driver graphdriver.Driver - execDriver execdriver.Driver - statsCollector *statsCollector - defaultLogConfig runconfig.LogConfig - RegistryService *registry.Service - EventsService *events.Events - netController libnetwork.NetworkController - volumes *store.VolumeStore - discoveryWatcher discovery.Watcher - root string - shutdown bool - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap + ID string + repository string + sysInitPath string + containers *contStore + execCommands *exec.Store + tagStore tag.Store + distributionPool *distribution.Pool + distributionMetadataStore dmetadata.Store + trustKey libtrust.PrivateKey + idIndex *truncindex.TruncIndex + configStore *Config + containerGraphDB *graphdb.Database + driver graphdriver.Driver + execDriver execdriver.Driver + statsCollector *statsCollector + defaultLogConfig runconfig.LogConfig + RegistryService *registry.Service + EventsService *events.Events + netController libnetwork.NetworkController + volumes *store.VolumeStore + discoveryWatcher discovery.Watcher + root string + shutdown bool + uidMaps []idtools.IDMap + gidMaps []idtools.IDMap + layerStore layer.Store + imageStore image.Store } // Get looks for a container using the provided information, which could be @@ -229,9 +250,7 @@ func (daemon *Daemon) Register(container *Container) error { container.unmountIpcMounts(mount.Unmount) - if err := daemon.Unmount(container); err != nil { - logrus.Debugf("unmount error %s", err) - } + daemon.Unmount(container) if err := container.toDiskLocking(); err != nil { logrus.Errorf("Error saving stopped state to disk: %v", err) } @@ -456,7 +475,7 @@ func (daemon *Daemon) getEntrypointAndArgs(configEntrypoint *stringutils.StrSlic return cmdSlice[0], cmdSlice[1:] } -func (daemon *Daemon) newContainer(name string, config *runconfig.Config, imgID string) (*Container, error) { +func (daemon *Daemon) newContainer(name string, config *runconfig.Config, imgID image.ID) (*Container, error) { var ( id string err error @@ -542,7 +561,7 @@ func (daemon *Daemon) GetLabels(id string) map[string]string { return container.Config.Labels } - img, err := daemon.repositories.LookupImage(id) + img, err := daemon.GetImage(id) if err == nil { return img.ContainerConfig.Labels } @@ -702,8 +721,25 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo return nil, err } - logrus.Debug("Creating images graph") - g, err := graph.NewGraph(filepath.Join(config.Root, "graph"), d.driver, uidMaps, gidMaps) + imageRoot := filepath.Join(config.Root, "image", d.driver.String()) + fms, err := layer.NewFSMetadataStore(filepath.Join(imageRoot, "layerdb")) + if err != nil { + return nil, err + } + + d.layerStore, err = layer.NewStore(fms, d.driver) + if err != nil { + return nil, err + } + + distributionPool := distribution.NewPool() + + ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb")) + if err != nil { + return nil, err + } + + d.imageStore, err = image.NewImageStore(ifs, d.layerStore) if err != nil { return nil, err } @@ -725,23 +761,24 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo return nil, err } + distributionMetadataStore, err := dmetadata.NewFSMetadataStore(filepath.Join(imageRoot, "distribution")) + if err != nil { + return nil, err + } + eventsService := events.New() - logrus.Debug("Creating repository list") - tagCfg := &graph.TagStoreConfig{ - Graph: g, - Key: trustKey, - Registry: registryService, - Events: eventsService, - } - repositories, err := graph.NewTagStore(filepath.Join(config.Root, "repositories-"+d.driver.String()), tagCfg) + + tagStore, err := tag.NewTagStore(filepath.Join(imageRoot, "repositories.json")) if err != nil { - return nil, fmt.Errorf("Couldn't create Tag store repositories-%s: %s", d.driver.String(), err) + return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err) } - if restorer, ok := d.driver.(graphdriver.ImageRestorer); ok { - if _, err := restorer.RestoreCustomImages(repositories, g); err != nil { - return nil, fmt.Errorf("Couldn't restore custom images: %s", err) - } + if err := restoreCustomImage(d.driver, d.imageStore, d.layerStore, tagStore); err != nil { + return nil, fmt.Errorf("Couldn't restore custom images: %s", err) + } + + if err := v1.Migrate(config.Root, d.driver.String(), d.layerStore, d.imageStore, tagStore, distributionMetadataStore); err != nil { + return nil, err } // Discovery is only enabled when the daemon is launched with an address to advertise. When @@ -792,8 +829,10 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo d.repository = daemonRepo d.containers = &contStore{s: make(map[string]*Container)} d.execCommands = exec.NewStore() - d.graph = g - d.repositories = repositories + d.tagStore = tagStore + d.distributionPool = distributionPool + d.distributionMetadataStore = distributionMetadataStore + d.trustKey = trustKey d.idIndex = truncindex.NewTruncIndex([]string{}) d.configStore = config d.sysInitPath = sysInitPath @@ -910,28 +949,44 @@ func (daemon *Daemon) Shutdown() error { // Mount sets container.basefs // (is it not set coming in? why is it unset?) func (daemon *Daemon) Mount(container *Container) error { - dir, err := daemon.driver.Get(container.ID, container.getMountLabel()) + var layerID layer.ChainID + if container.ImageID != "" { + img, err := daemon.imageStore.Get(container.ImageID) + if err != nil { + return err + } + layerID = img.RootFS.ChainID() + } + rwlayer, err := daemon.layerStore.Mount(container.ID, layerID, container.getMountLabel(), daemon.setupInitLayer) if err != nil { - return fmt.Errorf("Error getting container %s from driver %s: %s", container.ID, daemon.driver, err) + return err + } + dir, err := rwlayer.Path() + if err != nil { + return err } + logrus.Debugf("container mounted via layerStore: %v", dir) if container.basefs != dir { // The mount path reported by the graph driver should always be trusted on Windows, since the // volume path for a given mounted layer may change over time. This should only be an error // on non-Windows operating systems. if container.basefs != "" && runtime.GOOS != "windows" { - daemon.driver.Put(container.ID) + daemon.Unmount(container) return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')", daemon.driver, container.ID, container.basefs, dir) } } - container.basefs = dir + container.basefs = dir // TODO: combine these fields + container.rwlayer = rwlayer return nil } // Unmount unsets the container base filesystem -func (daemon *Daemon) Unmount(container *Container) error { - return daemon.driver.Put(container.ID) +func (daemon *Daemon) Unmount(container *Container) { + if err := daemon.layerStore.Unmount(container.ID); err != nil { + logrus.Errorf("Error unmounting container %s: %s", container.ID, err) + } } // Run uses the execution driver to run a given container @@ -962,82 +1017,46 @@ func (daemon *Daemon) unsubscribeToContainerStats(c *Container, ch chan interfac } func (daemon *Daemon) changes(container *Container) ([]archive.Change, error) { - initID := fmt.Sprintf("%s-init", container.ID) - return daemon.driver.Changes(container.ID, initID) -} - -func (daemon *Daemon) diff(container *Container) (archive.Archive, error) { - initID := fmt.Sprintf("%s-init", container.ID) - return daemon.driver.Diff(container.ID, initID) + return daemon.layerStore.Changes(container.ID) } -func (daemon *Daemon) createRootfs(container *Container) error { - // Step 1: create the container directory. - // This doubles as a barrier to avoid race conditions. - rootUID, rootGID, err := idtools.GetRootUIDGID(daemon.uidMaps, daemon.gidMaps) - if err != nil { - return err +// TagImage creates a tag in the repository reponame, pointing to the image named +// imageName. If force is true, an existing tag with the same name may be +// overwritten. +func (daemon *Daemon) TagImage(newTag reference.Named, imageName string, force bool) error { + if _, isDigested := newTag.(reference.Digested); isDigested { + return errors.New("refusing to create a tag with a digest reference") } - if err := idtools.MkdirAs(container.root, 0700, rootUID, rootGID); err != nil { - return err + if newTag.Name() == string(digest.Canonical) { + return errors.New("refusing to create an ambiguous tag using digest algorithm as name") } - initID := fmt.Sprintf("%s-init", container.ID) - if err := daemon.driver.Create(initID, container.ImageID, container.getMountLabel()); err != nil { - return err - } - initPath, err := daemon.driver.Get(initID, "") + newTag = registry.NormalizeLocalReference(newTag) + imageID, err := daemon.GetImageID(imageName) if err != nil { return err } - - if err := setupInitLayer(initPath, rootUID, rootGID); err != nil { - if err := daemon.driver.Put(initID); err != nil { - logrus.Errorf("Failed to Put init layer: %v", err) - } - return err - } - - // We want to unmount init layer before we take snapshot of it - // for the actual container. - if err := daemon.driver.Put(initID); err != nil { - return err - } - - if err := daemon.driver.Create(container.ID, initID, ""); err != nil { - return err - } - return nil -} - -// Graph returns *graph.Graph which can be using for layers graph operations. -func (daemon *Daemon) Graph() *graph.Graph { - return daemon.graph -} - -// TagImage creates a tag in the repository reponame, pointing to the image named -// imageName. If force is true, an existing tag with the same name may be -// overwritten. -func (daemon *Daemon) TagImage(repoName, tag, imageName string, force bool) error { - if err := daemon.repositories.Tag(repoName, tag, imageName, force); err != nil { - return err - } - daemon.EventsService.Log("tag", utils.ImageReference(repoName, tag), "") - return nil + daemon.EventsService.Log("tag", newTag.String(), "") + return daemon.tagStore.Add(newTag, imageID, force) } // PullImage initiates a pull operation. image is the repository name to pull, and // tag may be either empty, or indicate a specific tag to pull. -func (daemon *Daemon) PullImage(image string, tag string, imagePullConfig *graph.ImagePullConfig) error { - return daemon.repositories.Pull(image, tag, imagePullConfig) -} +func (daemon *Daemon) PullImage(ref reference.Named, metaHeaders map[string][]string, authConfig *cliconfig.AuthConfig, outStream io.Writer) error { + imagePullConfig := &distribution.ImagePullConfig{ + MetaHeaders: metaHeaders, + AuthConfig: authConfig, + OutStream: outStream, + RegistryService: daemon.RegistryService, + EventsService: daemon.EventsService, + MetadataStore: daemon.distributionMetadataStore, + LayerStore: daemon.layerStore, + ImageStore: daemon.imageStore, + TagStore: daemon.tagStore, + Pool: daemon.distributionPool, + } -// ImportImage imports an image, getting the archived layer data either from -// inConfig (if src is "-"), or from a URI specified in src. Progress output is -// written to outStream. Repository and tag names can optionally be given in -// the repo and tag arguments, respectively. -func (daemon *Daemon) ImportImage(src, repo, tag, msg string, inConfig io.ReadCloser, outStream io.Writer, containerConfig *runconfig.Config) error { - return daemon.repositories.Import(src, repo, tag, msg, inConfig, outStream, containerConfig) + return distribution.Pull(ref, imagePullConfig) } // ExportImage exports a list of images to the given output stream. The @@ -1046,47 +1065,214 @@ func (daemon *Daemon) ImportImage(src, repo, tag, msg string, inConfig io.ReadCl // the same tag are exported. names is the set of tags to export, and // outStream is the writer which the images are written to. func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error { - return daemon.repositories.ImageExport(names, outStream) + imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStore, daemon.tagStore) + return imageExporter.Save(names, outStream) } // PushImage initiates a push operation on the repository named localName. -func (daemon *Daemon) PushImage(localName string, imagePushConfig *graph.ImagePushConfig) error { - return daemon.repositories.Push(localName, imagePushConfig) +func (daemon *Daemon) PushImage(ref reference.Named, metaHeaders map[string][]string, authConfig *cliconfig.AuthConfig, outStream io.Writer) error { + imagePushConfig := &distribution.ImagePushConfig{ + MetaHeaders: metaHeaders, + AuthConfig: authConfig, + OutStream: outStream, + RegistryService: daemon.RegistryService, + EventsService: daemon.EventsService, + MetadataStore: daemon.distributionMetadataStore, + LayerStore: daemon.layerStore, + ImageStore: daemon.imageStore, + TagStore: daemon.tagStore, + TrustKey: daemon.trustKey, + } + + return distribution.Push(ref, imagePushConfig) } // LookupImage looks up an image by name and returns it as an ImageInspect // structure. func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { - return daemon.repositories.Lookup(name) + img, err := daemon.GetImage(name) + if err != nil { + return nil, fmt.Errorf("No such image: %s", name) + } + + refs := daemon.tagStore.References(img.ID()) + repoTags := []string{} + repoDigests := []string{} + for _, ref := range refs { + switch ref.(type) { + case reference.Tagged: + repoTags = append(repoTags, ref.String()) + case reference.Digested: + repoDigests = append(repoDigests, ref.String()) + } + } + + var size int64 + var layerMetadata map[string]string + layerID := img.RootFS.ChainID() + if layerID != "" { + l, err := daemon.layerStore.Get(layerID) + if err != nil { + return nil, err + } + defer layer.ReleaseAndLog(daemon.layerStore, l) + size, err = l.Size() + if err != nil { + return nil, err + } + + layerMetadata, err = l.Metadata() + if err != nil { + return nil, err + } + } + + imageInspect := &types.ImageInspect{ + ID: img.ID().String(), + RepoTags: repoTags, + RepoDigests: repoDigests, + Parent: img.Parent.String(), + Comment: img.Comment, + Created: img.Created.Format(time.RFC3339Nano), + Container: img.Container, + ContainerConfig: &img.ContainerConfig, + DockerVersion: img.DockerVersion, + Author: img.Author, + Config: img.Config, + Architecture: img.Architecture, + Os: img.OS, + Size: size, + VirtualSize: size, // TODO: field unused, deprecate + } + + imageInspect.GraphDriver.Name = daemon.driver.String() + + imageInspect.GraphDriver.Data = layerMetadata + + return imageInspect, nil } // LoadImage uploads a set of images into the repository. This is the // complement of ImageExport. The input stream is an uncompressed tar // ball containing images and metadata. func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer) error { - return daemon.repositories.Load(inTar, outStream) -} - -// ListImages returns a filtered list of images. filterArgs is a JSON-encoded set -// of filter arguments which will be interpreted by pkg/parsers/filters. -// filter is a shell glob string applied to repository names. The argument -// named all controls whether all images in the graph are filtered, or just -// the heads. -func (daemon *Daemon) ListImages(filterArgs, filter string, all bool) ([]*types.Image, error) { - return daemon.repositories.Images(filterArgs, filter, all) + imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStore, daemon.tagStore) + return imageExporter.Load(inTar, outStream) } // ImageHistory returns a slice of ImageHistory structures for the specified image // name by walking the image lineage. func (daemon *Daemon) ImageHistory(name string) ([]*types.ImageHistory, error) { - return daemon.repositories.History(name) + img, err := daemon.GetImage(name) + if err != nil { + return nil, err + } + + history := []*types.ImageHistory{} + + layerCounter := 0 + rootFS := *img.RootFS + rootFS.DiffIDs = nil + + for _, h := range img.History { + var layerSize int64 + + if !h.EmptyLayer { + if len(img.RootFS.DiffIDs) <= layerCounter { + return nil, errors.New("too many non-empty layers in History section") + } + + rootFS.Append(img.RootFS.DiffIDs[layerCounter]) + l, err := daemon.layerStore.Get(rootFS.ChainID()) + if err != nil { + return nil, err + } + layerSize, err = l.DiffSize() + layer.ReleaseAndLog(daemon.layerStore, l) + if err != nil { + return nil, err + } + + layerCounter++ + } + + history = append([]*types.ImageHistory{{ + ID: "", + Created: h.Created.Unix(), + CreatedBy: h.CreatedBy, + Comment: h.Comment, + Size: layerSize, + }}, history...) + } + + // Fill in image IDs and tags + histImg := img + id := img.ID() + for _, h := range history { + h.ID = id.String() + + var tags []string + for _, r := range daemon.tagStore.References(id) { + if _, ok := r.(reference.NamedTagged); ok { + tags = append(tags, r.String()) + } + } + + h.Tags = tags + + id = histImg.Parent + if id == "" { + break + } + histImg, err = daemon.GetImage(id.String()) + if err != nil { + break + } + } + + return history, nil } -// GetImage returns pointer to an Image struct corresponding to the given -// name. The name can include an optional tag; otherwise the default tag will -// be used. -func (daemon *Daemon) GetImage(name string) (*image.Image, error) { - return daemon.repositories.LookupImage(name) +// GetImageID returns an image ID corresponding to the image referred to by +// refOrID. +func (daemon *Daemon) GetImageID(refOrID string) (image.ID, error) { + // Treat as an ID + if id, err := digest.ParseDigest(refOrID); err == nil { + return image.ID(id), nil + } + + // Treat it as a possible tag or digest reference + if ref, err := reference.ParseNamed(refOrID); err == nil { + ref = registry.NormalizeLocalReference(ref) + if id, err := daemon.tagStore.Get(ref); err == nil { + return id, nil + } + if tagged, ok := ref.(reference.Tagged); ok { + if id, err := daemon.imageStore.Search(tagged.Tag()); err == nil { + for _, namedRef := range daemon.tagStore.References(id) { + if namedRef.Name() == ref.Name() { + return id, nil + } + } + } + } + } + + // Search based on ID + if id, err := daemon.imageStore.Search(refOrID); err == nil { + return id, nil + } + + return "", ErrImageDoesNotExist{refOrID} +} + +// GetImage returns an image corresponding to the image referred to by refOrID. +func (daemon *Daemon) GetImage(refOrID string) (*image.Image, error) { + imgID, err := daemon.GetImageID(refOrID) + if err != nil { + return nil, err + } + return daemon.imageStore.Get(imgID) } func (daemon *Daemon) config() *Config { @@ -1132,33 +1318,23 @@ func (daemon *Daemon) GetRemappedUIDGID() (int, int) { // of the image with imgID, that had the same config when it was // created. nil is returned if a child cannot be found. An error is // returned if the parent image cannot be found. -func (daemon *Daemon) ImageGetCached(imgID string, config *runconfig.Config) (*image.Image, error) { - // for now just exit if imgID has no children. - // maybe parentRefs in graph could be used to store - // the Image obj children for faster lookup below but this can - // be quite memory hungry. - if !daemon.Graph().HasChildren(imgID) { - return nil, nil - } - +func (daemon *Daemon) ImageGetCached(imgID image.ID, config *runconfig.Config) (*image.Image, error) { // Retrieve all images - images := daemon.Graph().Map() + imgs := daemon.Map() - // Store the tree in a map of map (map[parentId][childId]) - imageMap := make(map[string]map[string]struct{}) - for _, img := range images { - if _, exists := imageMap[img.Parent]; !exists { - imageMap[img.Parent] = make(map[string]struct{}) + var siblings []image.ID + for id, img := range imgs { + if img.Parent == imgID { + siblings = append(siblings, id) } - imageMap[img.Parent][img.ID] = struct{}{} } // Loop on the children of the given image and check the config var match *image.Image - for elem := range imageMap[imgID] { - img, ok := images[elem] + for _, id := range siblings { + img, ok := imgs[id] if !ok { - return nil, fmt.Errorf("unable to find image %q", elem) + return nil, fmt.Errorf("unable to find image %q", id) } if runconfig.Compare(&img.ContainerConfig, config) { if match == nil || match.Created.Before(img.Created) { @@ -1179,6 +1355,12 @@ func tempDir(rootDir string, rootUID, rootGID int) (string, error) { } func (daemon *Daemon) setHostConfig(container *Container, hostConfig *runconfig.HostConfig) error { + container.Lock() + if err := parseSecurityOpt(container, hostConfig); err != nil { + container.Unlock() + return err + } + container.Unlock() // Do not lock while creating volumes since this could be calling out to external plugins // Don't want to block other actions, like `docker ps` because we're waiting on an external plugin @@ -1199,6 +1381,11 @@ func (daemon *Daemon) setHostConfig(container *Container, hostConfig *runconfig. return nil } +func (daemon *Daemon) setupInitLayer(initPath string) error { + rootUID, rootGID := daemon.GetRemappedUIDGID() + return setupInitLayer(initPath, rootUID, rootGID) +} + func setDefaultMtu(config *Config) { // do nothing if the config does not have the default 0 value. if config.Mtu != 0 { diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 15b6bc69227ac..520ccf087cdf3 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -14,12 +14,15 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" derr "github.com/docker/docker/errors" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" pblkiodev "github.com/docker/docker/pkg/blkiodev" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/sysinfo" "github.com/docker/docker/runconfig" + "github.com/docker/docker/tag" "github.com/docker/libnetwork" nwconfig "github.com/docker/libnetwork/config" "github.com/docker/libnetwork/drivers/bridge" @@ -601,9 +604,7 @@ func (daemon *Daemon) conditionalMountOnStart(container *Container) error { // conditionalUnmountOnCleanup is a platform specific helper function called // during the cleanup of a container to unmount. func (daemon *Daemon) conditionalUnmountOnCleanup(container *Container) { - if err := daemon.Unmount(container); err != nil { - logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err) - } + daemon.Unmount(container) } // getDefaultRouteMtu returns the MTU for the default route's interface. @@ -624,3 +625,8 @@ func getDefaultRouteMtu() (int, error) { } return 0, errNoDefaultRoute } + +func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, ts tag.Store) error { + // Unix has no custom images to register + return nil +} diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go index 219a5c3786a63..53dad672dc64c 100644 --- a/daemon/daemon_windows.go +++ b/daemon/daemon_windows.go @@ -1,12 +1,22 @@ package daemon import ( + "encoding/json" + "errors" "fmt" + "path/filepath" + "runtime" + "strings" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/reference" "github.com/docker/docker/daemon/graphdriver" + "github.com/docker/docker/dockerversion" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/tag" // register the windows graph driver - _ "github.com/docker/docker/daemon/graphdriver/windows" + "github.com/docker/docker/daemon/graphdriver/windows" "github.com/docker/docker/pkg/system" "github.com/docker/docker/runconfig" "github.com/docker/libnetwork" @@ -128,8 +138,71 @@ func (daemon *Daemon) conditionalMountOnStart(container *Container) error { func (daemon *Daemon) conditionalUnmountOnCleanup(container *Container) { // We do not unmount if a Hyper-V container if !container.hostConfig.Isolation.IsHyperV() { - if err := daemon.Unmount(container); err != nil { - logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err) + daemon.Unmount(container) + } +} + +func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, ts tag.Store) error { + if wd, ok := driver.(*windows.Driver); ok { + imageInfos, err := wd.GetCustomImageInfos() + if err != nil { + return err } + + // Convert imageData to valid image configuration + for i := range imageInfos { + name := strings.ToLower(imageInfos[i].Name) + + type registrar interface { + RegisterDiffID(graphID string, size int64) (layer.Layer, error) + } + r, ok := ls.(registrar) + if !ok { + return errors.New("Layerstore doesn't support RegisterDiffID") + } + if _, err := r.RegisterDiffID(imageInfos[i].ID, imageInfos[i].Size); err != nil { + return err + } + // layer is intentionally not released + + rootFS := image.NewRootFS() + rootFS.BaseLayer = filepath.Base(imageInfos[i].Path) + + // Create history for base layer + config, err := json.Marshal(&image.Image{ + V1Image: image.V1Image{ + DockerVersion: dockerversion.Version, + Architecture: runtime.GOARCH, + OS: runtime.GOOS, + Created: imageInfos[i].CreatedTime, + }, + RootFS: rootFS, + History: []image.History{}, + }) + + named, err := reference.ParseNamed(name) + if err != nil { + return err + } + + ref, err := reference.WithTag(named, imageInfos[i].Version) + if err != nil { + return err + } + + id, err := is.Create(config) + if err != nil { + return err + } + + if err := ts.Add(ref, id, true); err != nil { + return err + } + + logrus.Debugf("Registered base layer %s as %s", ref, id) + } + } + + return nil } diff --git a/daemon/daemonbuilder/builder.go b/daemon/daemonbuilder/builder.go index 0b5528f16fdfe..4a94d9489e11f 100644 --- a/daemon/daemonbuilder/builder.go +++ b/daemon/daemonbuilder/builder.go @@ -9,17 +9,16 @@ import ( "strings" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/reference" "github.com/docker/docker/api" "github.com/docker/docker/builder" "github.com/docker/docker/cliconfig" "github.com/docker/docker/daemon" - "github.com/docker/docker/graph" "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/progressreader" "github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/registry" @@ -44,15 +43,24 @@ func (d Docker) LookupImage(name string) (*image.Image, error) { // Pull tells Docker to pull image referenced by `name`. func (d Docker) Pull(name string) (*image.Image, error) { - remote, tag := parsers.ParseRepositoryTag(name) - if tag == "" { - tag = "latest" + ref, err := reference.ParseNamed(name) + if err != nil { + return nil, err + } + switch ref.(type) { + case reference.Tagged: + case reference.Digested: + default: + ref, err = reference.WithTag(ref, "latest") + if err != nil { + return nil, err + } } pullRegistryAuth := &cliconfig.AuthConfig{} if len(d.AuthConfigs) > 0 { // The request came with a full auth config file, we prefer to use that - repoInfo, err := d.Daemon.RegistryService.ResolveRepository(remote) + repoInfo, err := d.Daemon.RegistryService.ResolveRepository(ref) if err != nil { return nil, err } @@ -64,12 +72,7 @@ func (d Docker) Pull(name string) (*image.Image, error) { pullRegistryAuth = &resolvedConfig } - imagePullConfig := &graph.ImagePullConfig{ - AuthConfig: pullRegistryAuth, - OutStream: ioutils.NopWriteCloser(d.OutOld), - } - - if err := d.Daemon.PullImage(remote, tag, imagePullConfig); err != nil { + if err := d.Daemon.PullImage(ref, nil, pullRegistryAuth, ioutils.NopWriteCloser(d.OutOld)); err != nil { return nil, err } @@ -106,18 +109,20 @@ func (d Docker) Remove(id string, cfg *daemon.ContainerRmConfig) error { } // Commit creates a new Docker image from an existing Docker container. -func (d Docker) Commit(name string, cfg *daemon.ContainerCommitConfig) (*image.Image, error) { +func (d Docker) Commit(name string, cfg *daemon.ContainerCommitConfig) (string, error) { return d.Daemon.Commit(name, cfg) } // Retain retains an image avoiding it to be removed or overwritten until a corresponding Release() call. func (d Docker) Retain(sessionID, imgID string) { - d.Daemon.Graph().Retain(sessionID, imgID) + // FIXME: This will be solved with tags in client-side builder + //d.Daemon.Graph().Retain(sessionID, imgID) } // Release releases a list of images that were retained for the time of a build. func (d Docker) Release(sessionID string, activeImages []string) { - d.Daemon.Graph().Release(sessionID, activeImages...) + // FIXME: This will be solved with tags in client-side builder + //d.Daemon.Graph().Release(sessionID, activeImages...) } // Copy copies/extracts a source FileInfo to a destination path inside a container @@ -199,11 +204,11 @@ func (d Docker) Copy(c *daemon.Container, destPath string, src builder.FileInfo, // GetCachedImage returns a reference to a cached image whose parent equals `parent` // and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error. func (d Docker) GetCachedImage(imgID string, cfg *runconfig.Config) (string, error) { - cache, err := d.Daemon.ImageGetCached(imgID, cfg) + cache, err := d.Daemon.ImageGetCached(image.ID(imgID), cfg) if cache == nil || err != nil { return "", err } - return cache.ID, nil + return cache.ID().String(), nil } // Kill stops the container execution abruptly. @@ -218,7 +223,8 @@ func (d Docker) Mount(c *daemon.Container) error { // Unmount unmounts the root filesystem for the container. func (d Docker) Unmount(c *daemon.Container) error { - return d.Daemon.Unmount(c) + d.Daemon.Unmount(c) + return nil } // Start starts a container diff --git a/daemon/delete.go b/daemon/delete.go index 018513cfad1d4..c06a319c6baa1 100644 --- a/daemon/delete.go +++ b/daemon/delete.go @@ -1,12 +1,12 @@ package daemon import ( - "fmt" "os" "path" "github.com/Sirupsen/logrus" derr "github.com/docker/docker/errors" + "github.com/docker/docker/layer" volumestore "github.com/docker/docker/volume/store" ) @@ -119,15 +119,12 @@ func (daemon *Daemon) rm(container *Container, forceRemove bool) (err error) { logrus.Debugf("Unable to remove container from link graph: %s", err) } - if err = daemon.driver.Remove(container.ID); err != nil { + metadata, err := daemon.layerStore.DeleteMount(container.ID) + layer.LogReleaseMetadata(metadata) + if err != nil { return derr.ErrorCodeRmDriverFS.WithArgs(daemon.driver, container.ID, err) } - initID := fmt.Sprintf("%s-init", container.ID) - if err := daemon.driver.Remove(initID); err != nil { - return derr.ErrorCodeRmInit.WithArgs(daemon.driver, initID, err) - } - if err = os.RemoveAll(container.root); err != nil { return derr.ErrorCodeRmFS.WithArgs(container.ID, err) } diff --git a/daemon/errors.go b/daemon/errors.go index 45a4882085197..8ab411c0f074a 100644 --- a/daemon/errors.go +++ b/daemon/errors.go @@ -3,21 +3,25 @@ package daemon import ( "strings" + "github.com/docker/distribution/reference" derr "github.com/docker/docker/errors" - "github.com/docker/docker/graph/tags" - "github.com/docker/docker/pkg/parsers" + tagpkg "github.com/docker/docker/tag" ) -func (d *Daemon) graphNotExistToErrcode(imageName string, err error) error { - if d.Graph().IsNotExist(err, imageName) { - if strings.Contains(imageName, "@") { - return derr.ErrorCodeNoSuchImageHash.WithArgs(imageName) +func (d *Daemon) imageNotExistToErrcode(err error) error { + if dne, isDNE := err.(ErrImageDoesNotExist); isDNE { + if strings.Contains(dne.RefOrID, "@") { + return derr.ErrorCodeNoSuchImageHash.WithArgs(dne.RefOrID) } - img, tag := parsers.ParseRepositoryTag(imageName) - if tag == "" { - tag = tags.DefaultTag + tag := tagpkg.DefaultTag + ref, err := reference.ParseNamed(dne.RefOrID) + if err != nil { + return derr.ErrorCodeNoSuchImageTag.WithArgs(dne.RefOrID, tag) } - return derr.ErrorCodeNoSuchImageTag.WithArgs(img, tag) + if tagged, isTagged := ref.(reference.Tagged); isTagged { + tag = tagged.Tag() + } + return derr.ErrorCodeNoSuchImageTag.WithArgs(ref.Name(), tag) } return err } diff --git a/daemon/events/filter.go b/daemon/events/filter.go index 49bbbe8a05c37..ae7fba93d12c3 100644 --- a/daemon/events/filter.go +++ b/daemon/events/filter.go @@ -1,8 +1,8 @@ package events import ( + "github.com/docker/distribution/reference" "github.com/docker/docker/pkg/jsonmessage" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/parsers/filters" ) @@ -38,8 +38,11 @@ func (ef *Filter) isLabelFieldIncluded(id string) bool { // against the stripped repo name without any tags. func (ef *Filter) isImageIncluded(eventID string, eventFrom string) bool { stripTag := func(image string) string { - repo, _ := parsers.ParseRepositoryTag(image) - return repo + ref, err := reference.ParseNamed(image) + if err != nil { + return image + } + return ref.Name() } return isFieldIncluded(eventID, ef.filter["image"]) || diff --git a/daemon/graphdriver/fsdiff.go b/daemon/graphdriver/fsdiff.go index ea0b053aa38a1..5a3493250be8a 100644 --- a/daemon/graphdriver/fsdiff.go +++ b/daemon/graphdriver/fsdiff.go @@ -1,5 +1,3 @@ -// +build daemon - package graphdriver import ( @@ -13,6 +11,12 @@ import ( "github.com/docker/docker/pkg/ioutils" ) +var ( + // ApplyUncompressedLayer defines the unpack method used by the graph + // driver. + ApplyUncompressedLayer = chrootarchive.ApplyUncompressedLayer +) + // NaiveDiffDriver takes a ProtoDriver and adds the // capability of the Diffing methods which it may or may not // support on its own. See the comment on the exported @@ -129,7 +133,7 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff archive.Reader) (s GIDMaps: gdw.gidMaps} start := time.Now().UTC() logrus.Debugf("Start untar layer") - if size, err = chrootarchive.ApplyUncompressedLayer(layerFs, diff, options); err != nil { + if size, err = ApplyUncompressedLayer(layerFs, diff, options); err != nil { return } logrus.Debugf("Untar time: %vs", time.Now().UTC().Sub(start).Seconds()) diff --git a/daemon/graphdriver/imagerestorer.go b/daemon/graphdriver/imagerestorer.go deleted file mode 100644 index d6592203b5c14..0000000000000 --- a/daemon/graphdriver/imagerestorer.go +++ /dev/null @@ -1,31 +0,0 @@ -package graphdriver - -import ( - "io" - - "github.com/docker/docker/image" -) - -// NOTE: These interfaces are used for implementing specific features of the Windows -// graphdriver implementation. The current versions are a short-term solution and -// likely to change or possibly be eliminated, so avoid using them outside of the Windows -// graphdriver code. - -// ImageRestorer interface allows the implementer to add a custom image to -// the graph and tagstore. -type ImageRestorer interface { - RestoreCustomImages(tagger Tagger, recorder Recorder) ([]string, error) -} - -// Tagger is an interface that exposes the TagStore.Tag function without needing -// to import graph. -type Tagger interface { - Tag(repoName, tag, imageName string, force bool) error -} - -// Recorder is an interface that exposes the Graph.Register and Graph.Exists -// functions without needing to import graph. -type Recorder interface { - Exists(id string) bool - Register(img image.Descriptor, layerData io.Reader) error -} diff --git a/daemon/graphdriver/plugin.go b/daemon/graphdriver/plugin.go index e816cd0d8464f..d63161b074844 100644 --- a/daemon/graphdriver/plugin.go +++ b/daemon/graphdriver/plugin.go @@ -1,5 +1,4 @@ // +build experimental -// +build daemon package graphdriver diff --git a/daemon/graphdriver/proxy.go b/daemon/graphdriver/proxy.go index 35a06da415480..47128473b0043 100644 --- a/daemon/graphdriver/proxy.go +++ b/daemon/graphdriver/proxy.go @@ -1,5 +1,4 @@ // +build experimental -// +build daemon package graphdriver diff --git a/daemon/graphdriver/vfs/driver.go b/daemon/graphdriver/vfs/driver.go index d2b807abc1294..ef90f19142255 100644 --- a/daemon/graphdriver/vfs/driver.go +++ b/daemon/graphdriver/vfs/driver.go @@ -1,5 +1,3 @@ -// +build daemon - package vfs import ( @@ -14,6 +12,11 @@ import ( "github.com/opencontainers/runc/libcontainer/label" ) +var ( + // CopyWithTar defines the copy method to use. + CopyWithTar = chrootarchive.CopyWithTar +) + func init() { graphdriver.Register("vfs", Init) } @@ -89,7 +92,7 @@ func (d *Driver) Create(id, parent, mountLabel string) error { if err != nil { return fmt.Errorf("%s: %s", parent, err) } - if err := chrootarchive.CopyWithTar(parentDir, dir); err != nil { + if err := CopyWithTar(parentDir, dir); err != nil { return err } return nil diff --git a/daemon/graphdriver/vfs/driver_unsupported.go b/daemon/graphdriver/vfs/driver_unsupported.go deleted file mode 100644 index 474a4c5c18f24..0000000000000 --- a/daemon/graphdriver/vfs/driver_unsupported.go +++ /dev/null @@ -1,3 +0,0 @@ -// +build !daemon - -package vfs diff --git a/daemon/graphdriver/windows/windows.go b/daemon/graphdriver/windows/windows.go index 007c6f907faf7..0594225268424 100644 --- a/daemon/graphdriver/windows/windows.go +++ b/daemon/graphdriver/windows/windows.go @@ -6,10 +6,10 @@ import ( "crypto/sha512" "encoding/json" "fmt" + "io" "io/ioutil" "os" "path/filepath" - "runtime" "strconv" "strings" "sync" @@ -17,8 +17,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" - "github.com/docker/docker/dockerversion" - "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/idtools" @@ -40,26 +38,6 @@ const ( filterDriver ) -// CustomImageDescriptor is an image descriptor for use by RestoreCustomImages -type customImageDescriptor struct { - img *image.Image -} - -// ID returns the image ID specified in the image structure. -func (img customImageDescriptor) ID() string { - return img.img.ID -} - -// Parent returns the parent ID - in this case, none -func (img customImageDescriptor) Parent() string { - return "" -} - -// MarshalConfig renders the image structure into JSON. -func (img customImageDescriptor) MarshalConfig() ([]byte, error) { - return json.Marshal(img.img) -} - // Driver represents a windows graph driver. type Driver struct { // info stores the shim driver information @@ -195,7 +173,7 @@ func (d *Driver) Remove(id string) error { if err != nil { return err } - + os.RemoveAll(filepath.Join(d.info.HomeDir, "sysfile-backups", rID)) // ok to fail return hcsshim.DestroyLayer(d.info, rID) } @@ -402,22 +380,27 @@ func (d *Driver) DiffSize(id, parent string) (size int64, err error) { return archive.ChangesSize(layerFs, changes), nil } -// RestoreCustomImages adds any auto-detected OS specific images to the tag and graph store. -func (d *Driver) RestoreCustomImages(tagger graphdriver.Tagger, recorder graphdriver.Recorder) (imageIDs []string, err error) { +// CustomImageInfo is the object returned by the driver describing the base +// image. +type CustomImageInfo struct { + ID string + Name string + Version string + Path string + Size int64 + CreatedTime time.Time +} + +// GetCustomImageInfos returns the image infos for window specific +// base images which should always be present. +func (d *Driver) GetCustomImageInfos() ([]CustomImageInfo, error) { strData, err := hcsshim.GetSharedBaseImages() if err != nil { return nil, fmt.Errorf("Failed to restore base images: %s", err) } - type customImageInfo struct { - Name string - Version string - Path string - Size int64 - CreatedTime time.Time - } type customImageInfoList struct { - Images []customImageInfo + Images []CustomImageInfo } var infoData customImageInfoList @@ -428,43 +411,28 @@ func (d *Driver) RestoreCustomImages(tagger graphdriver.Tagger, recorder graphdr return nil, err } + var images []CustomImageInfo + for _, imageData := range infoData.Images { - _, folderName := filepath.Split(imageData.Path) + folderName := filepath.Base(imageData.Path) // Use crypto hash of the foldername to generate a docker style id. h := sha512.Sum384([]byte(folderName)) id := fmt.Sprintf("%x", h[:32]) - if !recorder.Exists(id) { - // Register the image. - img := &image.Image{ - ID: id, - Created: imageData.CreatedTime, - DockerVersion: dockerversion.Version, - Architecture: runtime.GOARCH, - OS: runtime.GOOS, - Size: imageData.Size, - } - - if err := recorder.Register(customImageDescriptor{img}, nil); err != nil { - return nil, err - } - - // Create tags for the new image. - if err := tagger.Tag(strings.ToLower(imageData.Name), imageData.Version, img.ID, true); err != nil { - return nil, err - } - - // Create the alternate ID file. - if err := d.setID(img.ID, folderName); err != nil { - return nil, err - } - - imageIDs = append(imageIDs, img.ID) + if err := d.Create(id, "", ""); err != nil { + return nil, err } + // Create the alternate ID file. + if err := d.setID(id, folderName); err != nil { + return nil, err + } + + imageData.ID = id + images = append(images, imageData) } - return imageIDs, nil + return images, nil } // GetMetadata returns custom driver information. @@ -533,6 +501,10 @@ func (d *Driver) importLayer(id string, layerData archive.Reader, parentLayerPat if size, err = chrootarchive.ApplyLayer(tempFolder, layerData); err != nil { return } + err = copySysFiles(tempFolder, filepath.Join(d.info.HomeDir, "sysfile-backups", id)) + if err != nil { + return + } logrus.Debugf("Untar time: %vs", time.Now().UTC().Sub(start).Seconds()) if err = hcsshim.ImportLayer(d.info, id, tempFolder, parentLayerPaths); err != nil { @@ -596,3 +568,103 @@ func (d *Driver) setLayerChain(id string, chain []string) error { return nil } + +// DiffPath returns a directory that contains files needed to construct layer diff. +func (d *Driver) DiffPath(id string) (path string, release func() error, err error) { + id, err = d.resolveID(id) + if err != nil { + return + } + + // Getting the layer paths must be done outside of the lock. + layerChain, err := d.getLayerChain(id) + if err != nil { + return + } + + layerFolder := d.dir(id) + tempFolder := layerFolder + "-" + strconv.FormatUint(uint64(random.Rand.Uint32()), 10) + if err = os.MkdirAll(tempFolder, 0755); err != nil { + logrus.Errorf("Could not create %s %s", tempFolder, err) + return + } + + defer func() { + if err != nil { + _, folderName := filepath.Split(tempFolder) + if err2 := hcsshim.DestroyLayer(d.info, folderName); err2 != nil { + logrus.Warnf("Couldn't clean-up tempFolder: %s %s", tempFolder, err2) + } + } + }() + + if err = hcsshim.ExportLayer(d.info, id, tempFolder, layerChain); err != nil { + return + } + + err = copySysFiles(filepath.Join(d.info.HomeDir, "sysfile-backups", id), tempFolder) + if err != nil { + return + } + + return tempFolder, func() error { + // TODO: activate layers and release here? + _, folderName := filepath.Split(tempFolder) + return hcsshim.DestroyLayer(d.info, folderName) + }, nil +} + +var sysFileWhiteList = []string{ + "Hives\\*", + "Files\\BOOTNXT", + "tombstones.txt", +} + +// note this only handles files +func copySysFiles(src string, dest string) error { + if err := os.MkdirAll(dest, 0700); err != nil { + return err + } + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + for _, sysfile := range sysFileWhiteList { + if matches, err := filepath.Match(sysfile, rel); err != nil || !matches { + continue + } + + fi, err := os.Lstat(path) + if err != nil { + return err + } + + if !fi.Mode().IsRegular() { + continue + } + + targetPath := filepath.Join(dest, rel) + if err = os.MkdirAll(filepath.Dir(targetPath), 0700); err != nil { + return err + } + + in, err := os.Open(path) + if err != nil { + return err + } + out, err := os.Create(targetPath) + if err != nil { + in.Close() + return err + } + _, err = io.Copy(out, in) + in.Close() + out.Close() + if err != nil { + return err + } + } + return nil + }) +} diff --git a/daemon/image_delete.go b/daemon/image_delete.go index ca8b82a1f3380..3847be1ca370e 100644 --- a/daemon/image_delete.go +++ b/daemon/image_delete.go @@ -4,13 +4,12 @@ import ( "fmt" "strings" + "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" derr "github.com/docker/docker/errors" - "github.com/docker/docker/graph/tags" "github.com/docker/docker/image" - "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/stringid" - "github.com/docker/docker/utils" + tagpkg "github.com/docker/docker/tag" ) // ImageDelete deletes the image referenced by the given imageRef from this @@ -53,39 +52,46 @@ import ( func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDelete, error) { records := []types.ImageDelete{} - img, err := daemon.repositories.LookupImage(imageRef) + imgID, err := daemon.GetImageID(imageRef) if err != nil { - return nil, daemon.graphNotExistToErrcode(imageRef, err) + return nil, daemon.imageNotExistToErrcode(err) } + repoRefs := daemon.tagStore.References(imgID) + var removedRepositoryRef bool - if !isImageIDPrefix(img.ID, imageRef) { + if !isImageIDPrefix(imgID.String(), imageRef) { // A repository reference was given and should be removed // first. We can only remove this reference if either force is // true, there are multiple repository references to this // image, or there are no containers using the given reference. - if !(force || daemon.imageHasMultipleRepositoryReferences(img.ID)) { - if container := daemon.getContainerUsingImage(img.ID); container != nil { + if !(force || len(repoRefs) > 1) { + if container := daemon.getContainerUsingImage(imgID); container != nil { // If we removed the repository reference then // this image would remain "dangling" and since // we really want to avoid that the client must // explicitly force its removal. - return nil, derr.ErrorCodeImgDelUsed.WithArgs(imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(img.ID)) + return nil, derr.ErrorCodeImgDelUsed.WithArgs(imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(imgID.String())) } } - parsedRef, err := daemon.removeImageRef(imageRef) + parsedRef, err := reference.ParseNamed(imageRef) if err != nil { return nil, err } - untaggedRecord := types.ImageDelete{Untagged: parsedRef} + parsedRef, err = daemon.removeImageRef(parsedRef) + if err != nil { + return nil, err + } + + untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()} - daemon.EventsService.Log("untag", img.ID, "") + daemon.EventsService.Log("untag", imgID.String(), "") records = append(records, untaggedRecord) // If has remaining references then untag finishes the remove - if daemon.repositories.HasReferences(img) { + if len(repoRefs) > 1 { return records, nil } @@ -95,38 +101,39 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I // repository reference to the image then we will want to // remove that reference. // FIXME: Is this the behavior we want? - repoRefs := daemon.repositories.ByID()[img.ID] if len(repoRefs) == 1 { parsedRef, err := daemon.removeImageRef(repoRefs[0]) if err != nil { return nil, err } - untaggedRecord := types.ImageDelete{Untagged: parsedRef} + untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()} - daemon.EventsService.Log("untag", img.ID, "") + daemon.EventsService.Log("untag", imgID.String(), "") records = append(records, untaggedRecord) } } - return records, daemon.imageDeleteHelper(img, &records, force, prune, removedRepositoryRef) + return records, daemon.imageDeleteHelper(imgID, &records, force, prune, removedRepositoryRef) } // isImageIDPrefix returns whether the given possiblePrefix is a prefix of the // given imageID. func isImageIDPrefix(imageID, possiblePrefix string) bool { - return strings.HasPrefix(imageID, possiblePrefix) -} + if strings.HasPrefix(imageID, possiblePrefix) { + return true + } + + if i := strings.IndexRune(imageID, ':'); i >= 0 { + return strings.HasPrefix(imageID[i+1:], possiblePrefix) + } -// imageHasMultipleRepositoryReferences returns whether there are multiple -// repository references to the given imageID. -func (daemon *Daemon) imageHasMultipleRepositoryReferences(imageID string) bool { - return len(daemon.repositories.ByID()[imageID]) > 1 + return false } // getContainerUsingImage returns a container that was created using the given // imageID. Returns nil if there is no such container. -func (daemon *Daemon) getContainerUsingImage(imageID string) *Container { +func (daemon *Daemon) getContainerUsingImage(imageID image.ID) *Container { for _, container := range daemon.List() { if container.ImageID == imageID { return container @@ -141,18 +148,24 @@ func (daemon *Daemon) getContainerUsingImage(imageID string) *Container { // repositoryRef must not be an image ID but a repository name followed by an // optional tag or digest reference. If tag or digest is omitted, the default // tag is used. Returns the resolved image reference and an error. -func (daemon *Daemon) removeImageRef(repositoryRef string) (string, error) { - repository, ref := parsers.ParseRepositoryTag(repositoryRef) - if ref == "" { - ref = tags.DefaultTag +func (daemon *Daemon) removeImageRef(ref reference.Named) (reference.Named, error) { + switch ref.(type) { + case reference.Tagged: + case reference.Digested: + default: + var err error + ref, err = reference.WithTag(ref, tagpkg.DefaultTag) + if err != nil { + return nil, err + } } // Ignore the boolean value returned, as far as we're concerned, this // is an idempotent operation and it's okay if the reference didn't // exist in the first place. - _, err := daemon.repositories.Delete(repository, ref) + _, err := daemon.tagStore.Delete(ref) - return utils.ImageReference(repository, ref), err + return ref, err } // removeAllReferencesToImageID attempts to remove every reference to the given @@ -160,8 +173,8 @@ func (daemon *Daemon) removeImageRef(repositoryRef string) (string, error) { // on the first encountered error. Removed references are logged to this // daemon's event service. An "Untagged" types.ImageDelete is added to the // given list of records. -func (daemon *Daemon) removeAllReferencesToImageID(imgID string, records *[]types.ImageDelete) error { - imageRefs := daemon.repositories.ByID()[imgID] +func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDelete) error { + imageRefs := daemon.tagStore.References(imgID) for _, imageRef := range imageRefs { parsedRef, err := daemon.removeImageRef(imageRef) @@ -169,9 +182,9 @@ func (daemon *Daemon) removeAllReferencesToImageID(imgID string, records *[]type return err } - untaggedRecord := types.ImageDelete{Untagged: parsedRef} + untaggedRecord := types.ImageDelete{Untagged: parsedRef.String()} - daemon.EventsService.Log("untag", imgID, "") + daemon.EventsService.Log("untag", imgID.String(), "") *records = append(*records, untaggedRecord) } @@ -182,7 +195,7 @@ func (daemon *Daemon) removeAllReferencesToImageID(imgID string, records *[]type // Implements the error interface. type imageDeleteConflict struct { hard bool - imgID string + imgID image.ID message string } @@ -194,7 +207,7 @@ func (idc *imageDeleteConflict) Error() string { forceMsg = "must be forced" } - return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", stringid.TruncateID(idc.imgID), forceMsg, idc.message) + return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", stringid.TruncateID(idc.imgID.String()), forceMsg, idc.message) } // imageDeleteHelper attempts to delete the given image from this daemon. If @@ -208,11 +221,11 @@ func (idc *imageDeleteConflict) Error() string { // conflict is encountered, it will be returned immediately without deleting // the image. If quiet is true, any encountered conflicts will be ignored and // the function will return nil immediately without deleting the image. -func (daemon *Daemon) imageDeleteHelper(img *image.Image, records *[]types.ImageDelete, force, prune, quiet bool) error { +func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDelete, force, prune, quiet bool) error { // First, determine if this image has any conflicts. Ignore soft conflicts // if force is true. - if conflict := daemon.checkImageDeleteConflict(img, force); conflict != nil { - if quiet && !daemon.imageIsDangling(img) { + if conflict := daemon.checkImageDeleteConflict(imgID, force); conflict != nil { + if quiet && !daemon.imageIsDangling(imgID) { // Ignore conflicts UNLESS the image is "dangling" in // which case we want the user to know. return nil @@ -223,33 +236,38 @@ func (daemon *Daemon) imageDeleteHelper(img *image.Image, records *[]types.Image return conflict } + parent, err := daemon.imageStore.GetParent(imgID) + if err != nil { + // There may be no parent + parent = "" + } + // Delete all repository tag/digest references to this image. - if err := daemon.removeAllReferencesToImageID(img.ID, records); err != nil { + if err := daemon.removeAllReferencesToImageID(imgID, records); err != nil { return err } - if err := daemon.Graph().Delete(img.ID); err != nil { + removedLayers, err := daemon.imageStore.Delete(imgID) + if err != nil { return err } - daemon.EventsService.Log("delete", img.ID, "") - *records = append(*records, types.ImageDelete{Deleted: img.ID}) + daemon.EventsService.Log("delete", imgID.String(), "") + *records = append(*records, types.ImageDelete{Deleted: imgID.String()}) + for _, removedLayer := range removedLayers { + *records = append(*records, types.ImageDelete{Deleted: removedLayer.ChainID.String()}) + } - if !prune || img.Parent == "" { + if !prune || parent == "" { return nil } // We need to prune the parent image. This means delete it if there are // no tags/digests referencing it and there are no containers using it ( // either running or stopped). - parentImg, err := daemon.Graph().Get(img.Parent) - if err != nil { - return derr.ErrorCodeImgNoParent.WithArgs(err) - } - // Do not force prunings, but do so quietly (stopping on any encountered // conflicts). - return daemon.imageDeleteHelper(parentImg, records, false, true, true) + return daemon.imageDeleteHelper(parent, records, false, true, true) } // checkImageDeleteConflict determines whether there are any conflicts @@ -258,9 +276,9 @@ func (daemon *Daemon) imageDeleteHelper(img *image.Image, records *[]types.Image // using the image. A soft conflict is any tags/digest referencing the given // image or any stopped container using the image. If ignoreSoftConflicts is // true, this function will not check for soft conflict conditions. -func (daemon *Daemon) checkImageDeleteConflict(img *image.Image, ignoreSoftConflicts bool) *imageDeleteConflict { +func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, ignoreSoftConflicts bool) *imageDeleteConflict { // Check for hard conflicts first. - if conflict := daemon.checkImageDeleteHardConflict(img); conflict != nil { + if conflict := daemon.checkImageDeleteHardConflict(imgID); conflict != nil { return conflict } @@ -270,24 +288,15 @@ func (daemon *Daemon) checkImageDeleteConflict(img *image.Image, ignoreSoftConfl return nil } - return daemon.checkImageDeleteSoftConflict(img) + return daemon.checkImageDeleteSoftConflict(imgID) } -func (daemon *Daemon) checkImageDeleteHardConflict(img *image.Image) *imageDeleteConflict { - // Check if the image ID is being used by a pull or build. - if daemon.Graph().IsHeld(img.ID) { - return &imageDeleteConflict{ - hard: true, - imgID: img.ID, - message: "image is held by an ongoing pull or build", - } - } - +func (daemon *Daemon) checkImageDeleteHardConflict(imgID image.ID) *imageDeleteConflict { // Check if the image has any descendent images. - if daemon.Graph().HasChildren(img.ID) { + if len(daemon.imageStore.Children(imgID)) > 0 { return &imageDeleteConflict{ hard: true, - imgID: img.ID, + imgID: imgID, message: "image has dependent child images", } } @@ -299,9 +308,9 @@ func (daemon *Daemon) checkImageDeleteHardConflict(img *image.Image) *imageDelet continue } - if container.ImageID == img.ID { + if container.ImageID == imgID { return &imageDeleteConflict{ - imgID: img.ID, + imgID: imgID, hard: true, message: fmt.Sprintf("image is being used by running container %s", stringid.TruncateID(container.ID)), } @@ -311,11 +320,11 @@ func (daemon *Daemon) checkImageDeleteHardConflict(img *image.Image) *imageDelet return nil } -func (daemon *Daemon) checkImageDeleteSoftConflict(img *image.Image) *imageDeleteConflict { +func (daemon *Daemon) checkImageDeleteSoftConflict(imgID image.ID) *imageDeleteConflict { // Check if any repository tags/digest reference this image. - if daemon.repositories.HasReferences(img) { + if len(daemon.tagStore.References(imgID)) > 0 { return &imageDeleteConflict{ - imgID: img.ID, + imgID: imgID, message: "image is referenced in one or more repositories", } } @@ -327,9 +336,9 @@ func (daemon *Daemon) checkImageDeleteSoftConflict(img *image.Image) *imageDelet continue } - if container.ImageID == img.ID { + if container.ImageID == imgID { return &imageDeleteConflict{ - imgID: img.ID, + imgID: imgID, message: fmt.Sprintf("image is being used by stopped container %s", stringid.TruncateID(container.ID)), } } @@ -341,6 +350,6 @@ func (daemon *Daemon) checkImageDeleteSoftConflict(img *image.Image) *imageDelet // imageIsDangling returns whether the given image is "dangling" which means // that there are no repository references to the given image and it has no // child images. -func (daemon *Daemon) imageIsDangling(img *image.Image) bool { - return !(daemon.repositories.HasReferences(img) || daemon.Graph().HasChildren(img.ID)) +func (daemon *Daemon) imageIsDangling(imgID image.ID) bool { + return !(len(daemon.tagStore.References(imgID)) > 0 || len(daemon.imageStore.Children(imgID)) > 0) } diff --git a/daemon/images.go b/daemon/images.go new file mode 100644 index 0000000000000..b4c506ee052d8 --- /dev/null +++ b/daemon/images.go @@ -0,0 +1,163 @@ +package daemon + +import ( + "fmt" + "path" + "sort" + "strings" + + "github.com/docker/distribution/reference" + "github.com/docker/docker/api/types" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/parsers/filters" +) + +var acceptedImageFilterTags = map[string]struct{}{ + "dangling": {}, + "label": {}, +} + +// byCreated is a temporary type used to sort a list of images by creation +// time. +type byCreated []*types.Image + +func (r byCreated) Len() int { return len(r) } +func (r byCreated) Swap(i, j int) { r[i], r[j] = r[j], r[i] } +func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created } + +// Map returns a map of all images in the ImageStore +func (daemon *Daemon) Map() map[image.ID]*image.Image { + return daemon.imageStore.Map() +} + +// Images returns a filtered list of images. filterArgs is a JSON-encoded set +// of filter arguments which will be interpreted by pkg/parsers/filters. +// filter is a shell glob string applied to repository names. The argument +// named all controls whether all images in the graph are filtered, or just +// the heads. +func (daemon *Daemon) Images(filterArgs, filter string, all bool) ([]*types.Image, error) { + var ( + allImages map[image.ID]*image.Image + err error + danglingOnly = false + ) + + imageFilters, err := filters.FromParam(filterArgs) + if err != nil { + return nil, err + } + for name := range imageFilters { + if _, ok := acceptedImageFilterTags[name]; !ok { + return nil, fmt.Errorf("Invalid filter '%s'", name) + } + } + + if i, ok := imageFilters["dangling"]; ok { + for _, value := range i { + if v := strings.ToLower(value); v == "true" { + danglingOnly = true + } else if v != "false" { + return nil, fmt.Errorf("Invalid filter 'dangling=%s'", v) + } + } + } + + if danglingOnly { + allImages = daemon.imageStore.Heads() + } else { + allImages = daemon.imageStore.Map() + } + + images := []*types.Image{} + + var filterTagged bool + if filter != "" { + filterRef, err := reference.Parse(filter) + if err == nil { // parse error means wildcard repo + if _, ok := filterRef.(reference.Tagged); ok { + filterTagged = true + } + } + } + + for id, img := range allImages { + if _, ok := imageFilters["label"]; ok { + if img.Config == nil { + // Very old image that do not have image.Config (or even labels) + continue + } + // We are now sure image.Config is not nil + if !imageFilters.MatchKVList("label", img.Config.Labels) { + continue + } + } + + layerID := img.RootFS.ChainID() + var size int64 + if layerID != "" { + l, err := daemon.layerStore.Get(layerID) + if err != nil { + return nil, err + } + + size, err = l.Size() + layer.ReleaseAndLog(daemon.layerStore, l) + if err != nil { + return nil, err + } + } + + newImage := newImage(img, size) + + for _, ref := range daemon.tagStore.References(id) { + if filter != "" { // filter by tag/repo name + if filterTagged { // filter by tag, require full ref match + if ref.String() != filter { + continue + } + } else if matched, err := path.Match(filter, ref.Name()); !matched || err != nil { // name only match, FIXME: docs say exact + continue + } + } + if _, ok := ref.(reference.Digested); ok { + newImage.RepoDigests = append(newImage.RepoDigests, ref.String()) + } + if _, ok := ref.(reference.Tagged); ok { + newImage.RepoTags = append(newImage.RepoTags, ref.String()) + } + } + if newImage.RepoDigests == nil && newImage.RepoTags == nil { + if all || len(daemon.imageStore.Children(id)) == 0 { + if filter != "" { // skip images with no references if filtering by tag + continue + } + newImage.RepoDigests = []string{"@"} + newImage.RepoTags = []string{":"} + } else { + continue + } + } else if danglingOnly { + continue + } + + images = append(images, newImage) + } + + sort.Sort(sort.Reverse(byCreated(images))) + + return images, nil +} + +func newImage(image *image.Image, size int64) *types.Image { + newImage := new(types.Image) + newImage.ParentID = image.Parent.String() + newImage.ID = image.ID().String() + newImage.Created = image.Created.Unix() + newImage.Size = size + newImage.VirtualSize = size + if image.Config != nil { + newImage.Labels = image.Config.Labels + } + return newImage +} diff --git a/daemon/import.go b/daemon/import.go new file mode 100644 index 0000000000000..ac7247ae7d50f --- /dev/null +++ b/daemon/import.go @@ -0,0 +1,111 @@ +package daemon + +import ( + "encoding/json" + "io" + "net/http" + "net/url" + "runtime" + "time" + + "github.com/docker/distribution/reference" + "github.com/docker/docker/dockerversion" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/progressreader" + "github.com/docker/docker/pkg/streamformatter" + "github.com/docker/docker/runconfig" +) + +// ImportImage imports an image, getting the archived layer data either from +// inConfig (if src is "-"), or from a URI specified in src. Progress output is +// written to outStream. Repository and tag names can optionally be given in +// the repo and tag arguments, respectively. +func (daemon *Daemon) ImportImage(src string, newRef reference.Named, msg string, inConfig io.ReadCloser, outStream io.Writer, config *runconfig.Config) error { + var ( + sf = streamformatter.NewJSONStreamFormatter() + archive io.ReadCloser + resp *http.Response + ) + + if src == "-" { + archive = inConfig + } else { + inConfig.Close() + u, err := url.Parse(src) + if err != nil { + return err + } + if u.Scheme == "" { + u.Scheme = "http" + u.Host = src + u.Path = "" + } + outStream.Write(sf.FormatStatus("", "Downloading from %s", u)) + resp, err = httputils.Download(u.String()) + if err != nil { + return err + } + progressReader := progressreader.New(progressreader.Config{ + In: resp.Body, + Out: outStream, + Formatter: sf, + Size: resp.ContentLength, + NewLines: true, + ID: "", + Action: "Importing", + }) + archive = progressReader + } + + defer archive.Close() + if len(msg) == 0 { + msg = "Imported from " + src + } + // TODO: support windows baselayer? + l, err := daemon.layerStore.Register(archive, "") + if err != nil { + return err + } + defer layer.ReleaseAndLog(daemon.layerStore, l) + + created := time.Now().UTC() + imgConfig, err := json.Marshal(&image.Image{ + V1Image: image.V1Image{ + DockerVersion: dockerversion.Version, + Config: config, + Architecture: runtime.GOARCH, + OS: runtime.GOOS, + Created: created, + Comment: msg, + }, + RootFS: &image.RootFS{ + Type: "layers", + DiffIDs: []layer.DiffID{l.DiffID()}, + }, + History: []image.History{{ + Created: created, + Comment: msg, + }}, + }) + if err != nil { + return err + } + + id, err := daemon.imageStore.Create(imgConfig) + if err != nil { + return err + } + + // FIXME: connect with commit code and call tagstore directly + if newRef != nil { + if err := daemon.TagImage(newRef, id.String(), true); err != nil { + return err + } + } + + outStream.Write(sf.FormatStatus("", id.String())) + daemon.EventsService.Log("import", id.String(), "") + return nil +} diff --git a/daemon/info.go b/daemon/info.go index 921a236acdfeb..557f7799357c9 100644 --- a/daemon/info.go +++ b/daemon/info.go @@ -62,7 +62,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { v := &types.Info{ ID: daemon.ID, Containers: len(daemon.List()), - Images: len(daemon.Graph().Map()), + Images: len(daemon.imageStore.Map()), Driver: daemon.GraphDriver().String(), DriverStatus: daemon.GraphDriver().Status(), Plugins: daemon.showPluginsInfo(), diff --git a/daemon/inspect.go b/daemon/inspect.go index d17f2dca6c8b5..9adea0608c90b 100644 --- a/daemon/inspect.go +++ b/daemon/inspect.go @@ -8,6 +8,7 @@ import ( "github.com/docker/docker/api/types/versions/v1p20" "github.com/docker/docker/daemon/exec" "github.com/docker/docker/daemon/network" + "github.com/docker/docker/layer" ) // ContainerInspect returns low-level information about a @@ -124,7 +125,7 @@ func (daemon *Daemon) getInspectData(container *Container, size bool) (*types.Co Path: container.Path, Args: container.Args, State: containerState, - Image: container.ImageID, + Image: container.ImageID.String(), LogPath: container.LogPath, Name: container.Name, RestartCount: container.RestartCount, @@ -149,7 +150,18 @@ func (daemon *Daemon) getInspectData(container *Container, size bool) (*types.Co contJSONBase = setPlatformSpecificContainerFields(container, contJSONBase) contJSONBase.GraphDriver.Name = container.Driver - graphDriverData, err := daemon.driver.GetMetadata(container.ID) + + image, err := daemon.imageStore.Get(container.ImageID) + if err != nil { + return nil, err + } + l, err := daemon.layerStore.Get(image.RootFS.ChainID()) + if err != nil { + return nil, err + } + defer layer.ReleaseAndLog(daemon.layerStore, l) + + graphDriverData, err := l.Metadata() if err != nil { return nil, err } diff --git a/daemon/list.go b/daemon/list.go index 18152dc7aab12..b64103a0b7c19 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -9,7 +9,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" derr "github.com/docker/docker/errors" - "github.com/docker/docker/graph" "github.com/docker/docker/image" "github.com/docker/docker/pkg/graphdb" "github.com/docker/docker/pkg/nat" @@ -66,7 +65,7 @@ type listContext struct { // names is a list of container names to filter with names map[string][]string // images is a list of images to filter with - images map[string]bool + images map[image.ID]bool // filters is a collection of arguments to filter with, specified by the user filters filters.Args // exitAllowed is a list of exit codes allowed to filter with @@ -176,25 +175,24 @@ func (daemon *Daemon) foldFilter(config *ContainersConfig) (*listContext, error) } } - imagesFilter := map[string]bool{} + imagesFilter := map[image.ID]bool{} var ancestorFilter bool if ancestors, ok := psFilters["ancestor"]; ok { ancestorFilter = true - byParents := daemon.Graph().ByParent() // The idea is to walk the graph down the most "efficient" way. for _, ancestor := range ancestors { // First, get the imageId of the ancestor filter (yay) - image, err := daemon.repositories.LookupImage(ancestor) + id, err := daemon.GetImageID(ancestor) if err != nil { logrus.Warnf("Error while looking up for image %v", ancestor) continue } - if imagesFilter[ancestor] { + if imagesFilter[id] { // Already seen this ancestor, skip it continue } // Then walk down the graph and put the imageIds in imagesFilter - populateImageFilterByParents(imagesFilter, image.ID, byParents) + populateImageFilterByParents(imagesFilter, id, daemon.imageStore.Children) } } @@ -310,41 +308,29 @@ func includeContainerInList(container *Container, ctx *listContext) iterationAct return includeContainer } -func getImage(s *graph.TagStore, img, imgID string) (string, error) { - // both Image and ImageID is actually ids, nothing to guess - if strings.HasPrefix(imgID, img) { - return img, nil - } - id, err := s.GetID(img) - if err != nil { - if err == graph.ErrNameIsNotExist { - return imgID, nil - } - return "", err - } - if id != imgID { - return imgID, nil - } - return img, nil -} - // transformContainer generates the container type expected by the docker ps command. func (daemon *Daemon) transformContainer(container *Container, ctx *listContext) (*types.Container, error) { newC := &types.Container{ ID: container.ID, Names: ctx.names[container.ID], - ImageID: container.ImageID, + ImageID: container.ImageID.String(), } if newC.Names == nil { // Dead containers will often have no name, so make sure the response isn't null newC.Names = []string{} } - showImg, err := getImage(daemon.repositories, container.Config.Image, container.ImageID) - if err != nil { - return nil, err + image := container.Config.Image // if possible keep the original ref + if image != container.ImageID.String() { + id, err := daemon.GetImageID(image) + if _, isDNE := err.(ErrImageDoesNotExist); err != nil && !isDNE { + return nil, err + } + if err != nil || id != container.ImageID { + image = container.ImageID.String() + } } - newC.Image = showImg + newC.Image = image if len(container.Args) > 0 { args := []string{} @@ -433,12 +419,10 @@ func (daemon *Daemon) Volumes(filter string) ([]*types.Volume, error) { return volumesOut, nil } -func populateImageFilterByParents(ancestorMap map[string]bool, imageID string, byParents map[string][]*image.Image) { +func populateImageFilterByParents(ancestorMap map[image.ID]bool, imageID image.ID, getChildren func(image.ID) []image.ID) { if !ancestorMap[imageID] { - if images, ok := byParents[imageID]; ok { - for _, image := range images { - populateImageFilterByParents(ancestorMap, image.ID, byParents) - } + for _, id := range getChildren(imageID) { + populateImageFilterByParents(ancestorMap, id, getChildren) } ancestorMap[imageID] = true } diff --git a/errors/daemon.go b/errors/daemon.go index 1a13e74918472..72ddf278df265 100644 --- a/errors/daemon.go +++ b/errors/daemon.go @@ -832,15 +832,6 @@ var ( HTTPStatusCode: http.StatusInternalServerError, }) - // ErrorCodeRmInit is generated when we try to delete a container - // but failed deleting its init filesystem. - ErrorCodeRmInit = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMINIT", - Message: "Driver %s failed to remove init filesystem %s: %s", - Description: "While trying to delete a container, the driver failed to remove the init filesystem", - HTTPStatusCode: http.StatusInternalServerError, - }) - // ErrorCodeRmFS is generated when we try to delete a container // but failed deleting its filesystem. ErrorCodeRmFS = errcode.Register(errGroup, errcode.ErrorDescriptor{ diff --git a/integration-cli/docker_api_containers_test.go b/integration-cli/docker_api_containers_test.go index 216c20f2005c8..d092beb0bc2e0 100644 --- a/integration-cli/docker_api_containers_test.go +++ b/integration-cli/docker_api_containers_test.go @@ -553,7 +553,7 @@ func (s *DockerSuite) TestContainerApiCommit(c *check.C) { cName := "testapicommit" dockerCmd(c, "run", "--name="+cName, "busybox", "/bin/sh", "-c", "touch /test") - name := "TestContainerApiCommit" + name := "testcontainerapicommit" status, b, err := sockRequest("POST", "/commit?repo="+name+"&testtag=tag&container="+cName, nil) c.Assert(err, check.IsNil) c.Assert(status, check.Equals, http.StatusCreated) @@ -586,7 +586,7 @@ func (s *DockerSuite) TestContainerApiCommitWithLabelInConfig(c *check.C) { "Labels": map[string]string{"key1": "value1", "key2": "value2"}, } - name := "TestContainerApiCommitWithConfig" + name := "testcontainerapicommitwithconfig" status, b, err := sockRequest("POST", "/commit?repo="+name+"&container="+cName, config) c.Assert(err, check.IsNil) c.Assert(status, check.Equals, http.StatusCreated) diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 5b2ae74888110..d4b31bd38fe6f 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -4543,7 +4543,7 @@ func (s *DockerSuite) TestBuildInvalidTag(c *check.C) { _, out, err := buildImageWithOut(name, "FROM scratch\nMAINTAINER quux\n", true) // if the error doesnt check for illegal tag name, or the image is built // then this should fail - if !strings.Contains(out, "Illegal tag name") || strings.Contains(out, "Sending build context to Docker daemon") { + if !strings.Contains(out, "invalid reference format") || strings.Contains(out, "Sending build context to Docker daemon") { c.Fatalf("failed to stop before building. Error: %s, Output: %s", err, out) } } @@ -6377,7 +6377,7 @@ func (s *DockerSuite) TestBuildTagEvent(c *check.C) { select { case ev := <-ch: c.Assert(ev.Status, check.Equals, "tag") - c.Assert(ev.ID, check.Equals, "test:") + c.Assert(ev.ID, check.Equals, "test:latest") case <-time.After(time.Second): c.Fatal("The 'tag' event not heard from the server") } diff --git a/integration-cli/docker_cli_by_digest_test.go b/integration-cli/docker_cli_by_digest_test.go index 2d00605627d74..75c8b4fd6fa38 100644 --- a/integration-cli/docker_cli_by_digest_test.go +++ b/integration-cli/docker_cli_by_digest_test.go @@ -3,6 +3,8 @@ package main import ( "encoding/json" "fmt" + "os" + "path/filepath" "regexp" "strings" @@ -11,7 +13,6 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/pkg/integration/checker" "github.com/docker/docker/pkg/stringutils" - "github.com/docker/docker/utils" "github.com/go-check/check" ) @@ -32,7 +33,7 @@ func setupImageWithTag(c *check.C, tag string) (digest.Digest, error) { dockerCmd(c, "run", "-d", "-e", "digest=1", "--name", containerName, "busybox") // tag the image to upload it to the private registry - repoAndTag := utils.ImageReference(repoName, tag) + repoAndTag := repoName + ":" + tag out, _, err := dockerCmdWithError("commit", containerName, repoAndTag) c.Assert(err, checker.IsNil, check.Commentf("image tagging failed: %s", out)) @@ -438,6 +439,11 @@ func (s *DockerRegistrySuite) TestPullFailsWithAlteredLayer(c *check.C) { // Now try pulling that image by digest. We should get an error about // digest verification for the target layer digest. + // Remove distribution cache to force a re-pull of the blobs + if err := os.RemoveAll(filepath.Join(dockerBasePath, "image", s.d.storageDriver, "distribution")); err != nil { + c.Fatalf("error clearing distribution cache: %v", err) + } + // Pull from the registry using the @ reference. imageReference := fmt.Sprintf("%s@%s", repoName, manifestDigest) out, exitStatus, _ := dockerCmdWithError("pull", imageReference) diff --git a/integration-cli/docker_cli_create_test.go b/integration-cli/docker_cli_create_test.go index feb3fc055d509..cac5b92d5c235 100644 --- a/integration-cli/docker_cli_create_test.go +++ b/integration-cli/docker_cli_create_test.go @@ -14,6 +14,7 @@ import ( "github.com/docker/docker/pkg/integration/checker" "github.com/docker/docker/pkg/nat" + "github.com/docker/docker/pkg/stringid" "github.com/go-check/check" ) @@ -243,6 +244,41 @@ func (s *DockerSuite) TestCreateModeIpcContainer(c *check.C) { dockerCmd(c, "create", fmt.Sprintf("--ipc=container:%s", id), "busybox") } +func (s *DockerSuite) TestCreateByImageID(c *check.C) { + imageName := "testcreatebyimageid" + imageID, err := buildImage(imageName, + `FROM busybox + MAINTAINER dockerio`, + true) + if err != nil { + c.Fatal(err) + } + truncatedImageID := stringid.TruncateID(imageID) + + dockerCmd(c, "create", imageID) + dockerCmd(c, "create", truncatedImageID) + dockerCmd(c, "create", fmt.Sprintf("%s:%s", imageName, truncatedImageID)) + + // Ensure this fails + out, exit, _ := dockerCmdWithError("create", fmt.Sprintf("%s:%s", imageName, imageID)) + if exit == 0 { + c.Fatalf("expected non-zero exit code; received %d", exit) + } + + if expected := "invalid reference format"; !strings.Contains(out, expected) { + c.Fatalf(`Expected %q in output; got: %s`, expected, out) + } + + out, exit, _ = dockerCmdWithError("create", fmt.Sprintf("%s:%s", "wrongimage", truncatedImageID)) + if exit == 0 { + c.Fatalf("expected non-zero exit code; received %d", exit) + } + + if expected := "Unable to find image"; !strings.Contains(out, expected) { + c.Fatalf(`Expected %q in output; got: %s`, expected, out) + } +} + func (s *DockerTrustSuite) TestTrustedCreate(c *check.C) { repoName := s.setupTrustedImage(c, "trusted-create") diff --git a/integration-cli/docker_cli_external_graphdriver_unix_test.go b/integration-cli/docker_cli_external_graphdriver_unix_test.go index 4f87f60f0849e..3dcbe620ab7c3 100644 --- a/integration-cli/docker_cli_external_graphdriver_unix_test.go +++ b/integration-cli/docker_cli_external_graphdriver_unix_test.go @@ -325,6 +325,8 @@ func (s *DockerExternalGraphdriverSuite) TestExternalGraphDriver(c *check.C) { err = s.d.Stop() c.Assert(err, check.IsNil) + // Don't check s.ec.exists, because the daemon no longer calls the + // Exists function. c.Assert(s.ec.activations, check.Equals, 2) c.Assert(s.ec.init, check.Equals, 2) c.Assert(s.ec.creations >= 1, check.Equals, true) @@ -333,7 +335,6 @@ func (s *DockerExternalGraphdriverSuite) TestExternalGraphDriver(c *check.C) { c.Assert(s.ec.puts >= 1, check.Equals, true) c.Assert(s.ec.stats, check.Equals, 3) c.Assert(s.ec.cleanups, check.Equals, 2) - c.Assert(s.ec.exists >= 1, check.Equals, true) c.Assert(s.ec.applydiff >= 1, check.Equals, true) c.Assert(s.ec.changes, check.Equals, 1) c.Assert(s.ec.diffsize, check.Equals, 0) diff --git a/integration-cli/docker_cli_images_test.go b/integration-cli/docker_cli_images_test.go index 115b08a586475..42e3e46283721 100644 --- a/integration-cli/docker_cli_images_test.go +++ b/integration-cli/docker_cli_images_test.go @@ -98,9 +98,9 @@ func (s *DockerSuite) TestImagesFilterLabel(c *check.C) { out, _ := dockerCmd(c, "images", "--no-trunc", "-q", "-f", "label=match") out = strings.TrimSpace(out) - c.Assert(out, check.Matches, fmt.Sprintf("[\\s\\w]*%s[\\s\\w]*", image1ID)) - c.Assert(out, check.Matches, fmt.Sprintf("[\\s\\w]*%s[\\s\\w]*", image2ID)) - c.Assert(out, check.Not(check.Matches), fmt.Sprintf("[\\s\\w]*%s[\\s\\w]*", image3ID)) + c.Assert(out, check.Matches, fmt.Sprintf("[\\s\\w:]*%s[\\s\\w:]*", image1ID)) + c.Assert(out, check.Matches, fmt.Sprintf("[\\s\\w:]*%s[\\s\\w:]*", image2ID)) + c.Assert(out, check.Not(check.Matches), fmt.Sprintf("[\\s\\w:]*%s[\\s\\w:]*", image3ID)) out, _ = dockerCmd(c, "images", "--no-trunc", "-q", "-f", "label=match=me too") out = strings.TrimSpace(out) @@ -204,7 +204,7 @@ func (s *DockerSuite) TestImagesEnsureOnlyHeadsImagesShown(c *check.C) { // images shouldn't show non-heads images c.Assert(out, checker.Not(checker.Contains), intermediate) // images should contain final built images - c.Assert(out, checker.Contains, head[:12]) + c.Assert(out, checker.Contains, stringid.TruncateID(head)) } func (s *DockerSuite) TestImagesEnsureImagesFromScratchShown(c *check.C) { @@ -219,5 +219,5 @@ func (s *DockerSuite) TestImagesEnsureImagesFromScratchShown(c *check.C) { out, _ := dockerCmd(c, "images") // images should contain images built from scratch - c.Assert(out, checker.Contains, id[:12]) + c.Assert(out, checker.Contains, stringid.TruncateID(id)) } diff --git a/integration-cli/docker_cli_inspect_test.go b/integration-cli/docker_cli_inspect_test.go index a035f1a15b557..2e2c7b94aca41 100644 --- a/integration-cli/docker_cli_inspect_test.go +++ b/integration-cli/docker_cli_inspect_test.go @@ -23,7 +23,12 @@ func checkValidGraphDriver(c *check.C, name string) { func (s *DockerSuite) TestInspectImage(c *check.C) { testRequires(c, DaemonIsLinux) imageTest := "emptyfs" - imageTestID := "511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158" + // It is important that this ID remain stable. If a code change causes + // it to be different, this is equivalent to a cache bust when pulling + // a legacy-format manifest. If the check at the end of this function + // fails, fix the difference in the image serialization instead of + // updating this hash. + imageTestID := "sha256:11f64303f0f7ffdc71f001788132bca5346831939a956e3e975c93267d89a16d" id, err := inspectField(imageTest, "Id") c.Assert(err, checker.IsNil) diff --git a/integration-cli/docker_cli_pull_local_test.go b/integration-cli/docker_cli_pull_local_test.go index 2d5374bef9ba6..5771513eb9c59 100644 --- a/integration-cli/docker_cli_pull_local_test.go +++ b/integration-cli/docker_cli_pull_local_test.go @@ -159,3 +159,71 @@ func (s *DockerRegistrySuite) TestConcurrentPullMultipleTags(c *check.C) { c.Assert(strings.TrimSpace(out), check.Equals, "/bin/sh -c echo "+repo, check.Commentf("CMD did not contain /bin/sh -c echo %s; %s", repo, out)) } } + +// TestPullIDStability verifies that pushing an image and pulling it back +// preserves the image ID. +func (s *DockerRegistrySuite) TestPullIDStability(c *check.C) { + derivedImage := privateRegistryURL + "/dockercli/id-stability" + baseImage := "busybox" + + _, err := buildImage(derivedImage, fmt.Sprintf(` + FROM %s + ENV derived true + ENV asdf true + RUN dd if=/dev/zero of=/file bs=1024 count=1024 + CMD echo %s + `, baseImage, derivedImage), true) + if err != nil { + c.Fatal(err) + } + + originalID, err := getIDByName(derivedImage) + if err != nil { + c.Fatalf("error inspecting: %v", err) + } + dockerCmd(c, "push", derivedImage) + + // Pull + out, _ := dockerCmd(c, "pull", derivedImage) + if strings.Contains(out, "Pull complete") { + c.Fatalf("repull redownloaded a layer: %s", out) + } + + derivedIDAfterPull, err := getIDByName(derivedImage) + if err != nil { + c.Fatalf("error inspecting: %v", err) + } + + if derivedIDAfterPull != originalID { + c.Fatal("image's ID unexpectedly changed after a repush/repull") + } + + // Make sure the image runs correctly + out, _ = dockerCmd(c, "run", "--rm", derivedImage) + if strings.TrimSpace(out) != derivedImage { + c.Fatalf("expected %s; got %s", derivedImage, out) + } + + // Confirm that repushing and repulling does not change the computed ID + dockerCmd(c, "push", derivedImage) + dockerCmd(c, "rmi", derivedImage) + dockerCmd(c, "pull", derivedImage) + + derivedIDAfterPull, err = getIDByName(derivedImage) + if err != nil { + c.Fatalf("error inspecting: %v", err) + } + + if derivedIDAfterPull != originalID { + c.Fatal("image's ID unexpectedly changed after a repush/repull") + } + if err != nil { + c.Fatalf("error inspecting: %v", err) + } + + // Make sure the image still runs + out, _ = dockerCmd(c, "run", "--rm", derivedImage) + if strings.TrimSpace(out) != derivedImage { + c.Fatalf("expected %s; got %s", derivedImage, out) + } +} diff --git a/integration-cli/docker_cli_pull_test.go b/integration-cli/docker_cli_pull_test.go index ce74c7d7f2bc5..4f5f7281fb21e 100644 --- a/integration-cli/docker_cli_pull_test.go +++ b/integration-cli/docker_cli_pull_test.go @@ -1,11 +1,7 @@ package main import ( - "encoding/json" "fmt" - "io/ioutil" - "os" - "path/filepath" "regexp" "strings" "time" @@ -46,19 +42,19 @@ func (s *DockerHubPullSuite) TestPullFromCentralRegistry(c *check.C) { func (s *DockerHubPullSuite) TestPullNonExistingImage(c *check.C) { testRequires(c, DaemonIsLinux) for _, e := range []struct { - Image string + Repo string Alias string }{ - {"library/asdfasdf:foobar", "asdfasdf:foobar"}, - {"library/asdfasdf:foobar", "library/asdfasdf:foobar"}, - {"library/asdfasdf:latest", "asdfasdf"}, - {"library/asdfasdf:latest", "asdfasdf:latest"}, - {"library/asdfasdf:latest", "library/asdfasdf"}, - {"library/asdfasdf:latest", "library/asdfasdf:latest"}, + {"library/asdfasdf", "asdfasdf:foobar"}, + {"library/asdfasdf", "library/asdfasdf:foobar"}, + {"library/asdfasdf", "asdfasdf"}, + {"library/asdfasdf", "asdfasdf:latest"}, + {"library/asdfasdf", "library/asdfasdf"}, + {"library/asdfasdf", "library/asdfasdf:latest"}, } { out, err := s.CmdWithError("pull", e.Alias) c.Assert(err, checker.NotNil, check.Commentf("expected non-zero exit status when pulling non-existing image: %s", out)) - c.Assert(out, checker.Contains, fmt.Sprintf("Error: image %s not found", e.Image), check.Commentf("expected image not found error messages")) + c.Assert(out, checker.Contains, fmt.Sprintf("Error: image %s not found", e.Repo), check.Commentf("expected image not found error messages")) } } @@ -163,254 +159,3 @@ func (s *DockerHubPullSuite) TestPullClientDisconnect(c *check.C) { time.Sleep(500 * time.Millisecond) } } - -type idAndParent struct { - ID string - Parent string -} - -func inspectImage(c *check.C, imageRef string) idAndParent { - out, _ := dockerCmd(c, "inspect", imageRef) - var inspectOutput []idAndParent - err := json.Unmarshal([]byte(out), &inspectOutput) - if err != nil { - c.Fatal(err) - } - - return inspectOutput[0] -} - -func imageID(c *check.C, imageRef string) string { - return inspectImage(c, imageRef).ID -} - -func imageParent(c *check.C, imageRef string) string { - return inspectImage(c, imageRef).Parent -} - -// TestPullMigration verifies that pulling an image based on layers -// that already exists locally will reuse those existing layers. -func (s *DockerRegistrySuite) TestPullMigration(c *check.C) { - repoName := privateRegistryURL + "/dockercli/migration" - - baseImage := repoName + ":base" - _, err := buildImage(baseImage, fmt.Sprintf(` - FROM scratch - ENV IMAGE base - CMD echo %s - `, baseImage), true) - if err != nil { - c.Fatal(err) - } - - baseIDBeforePush := imageID(c, baseImage) - baseParentBeforePush := imageParent(c, baseImage) - - derivedImage := repoName + ":derived" - _, err = buildImage(derivedImage, fmt.Sprintf(` - FROM %s - CMD echo %s - `, baseImage, derivedImage), true) - if err != nil { - c.Fatal(err) - } - - derivedIDBeforePush := imageID(c, derivedImage) - - dockerCmd(c, "push", derivedImage) - - // Remove derived image from the local store - dockerCmd(c, "rmi", derivedImage) - - // Repull - dockerCmd(c, "pull", derivedImage) - - // Check that the parent of this pulled image is the original base - // image - derivedIDAfterPull1 := imageID(c, derivedImage) - derivedParentAfterPull1 := imageParent(c, derivedImage) - - if derivedIDAfterPull1 == derivedIDBeforePush { - c.Fatal("image's ID should have changed on after deleting and pulling") - } - - if derivedParentAfterPull1 != baseIDBeforePush { - c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush) - } - - // Confirm that repushing and repulling does not change the computed ID - dockerCmd(c, "push", derivedImage) - dockerCmd(c, "rmi", derivedImage) - dockerCmd(c, "pull", derivedImage) - - derivedIDAfterPull2 := imageID(c, derivedImage) - derivedParentAfterPull2 := imageParent(c, derivedImage) - - if derivedIDAfterPull2 != derivedIDAfterPull1 { - c.Fatal("image's ID unexpectedly changed after a repush/repull") - } - - if derivedParentAfterPull2 != baseIDBeforePush { - c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush) - } - - // Remove everything, repull, and make sure everything uses computed IDs - dockerCmd(c, "rmi", baseImage, derivedImage) - dockerCmd(c, "pull", derivedImage) - - derivedIDAfterPull3 := imageID(c, derivedImage) - derivedParentAfterPull3 := imageParent(c, derivedImage) - derivedGrandparentAfterPull3 := imageParent(c, derivedParentAfterPull3) - - if derivedIDAfterPull3 != derivedIDAfterPull1 { - c.Fatal("image's ID unexpectedly changed after a second repull") - } - - if derivedParentAfterPull3 == baseIDBeforePush { - c.Fatalf("pulled image's parent ID (%s) should not match base image's original ID (%s)", derivedParentAfterPull3, derivedIDBeforePush) - } - - if derivedGrandparentAfterPull3 == baseParentBeforePush { - c.Fatal("base image's parent ID should have been rewritten on pull") - } -} - -// TestPullMigrationRun verifies that pulling an image based on layers -// that already exists locally will result in an image that runs properly. -func (s *DockerRegistrySuite) TestPullMigrationRun(c *check.C) { - type idAndParent struct { - ID string - Parent string - } - - derivedImage := privateRegistryURL + "/dockercli/migration-run" - baseImage := "busybox" - - _, err := buildImage(derivedImage, fmt.Sprintf(` - FROM %s - RUN dd if=/dev/zero of=/file bs=1024 count=1024 - CMD echo %s - `, baseImage, derivedImage), true) - if err != nil { - c.Fatal(err) - } - - baseIDBeforePush := imageID(c, baseImage) - derivedIDBeforePush := imageID(c, derivedImage) - - dockerCmd(c, "push", derivedImage) - - // Remove derived image from the local store - dockerCmd(c, "rmi", derivedImage) - - // Repull - dockerCmd(c, "pull", derivedImage) - - // Check that this pulled image is based on the original base image - derivedIDAfterPull1 := imageID(c, derivedImage) - derivedParentAfterPull1 := imageParent(c, imageParent(c, derivedImage)) - - if derivedIDAfterPull1 == derivedIDBeforePush { - c.Fatal("image's ID should have changed on after deleting and pulling") - } - - if derivedParentAfterPull1 != baseIDBeforePush { - c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush) - } - - // Make sure the image runs correctly - out, _ := dockerCmd(c, "run", "--rm", derivedImage) - if strings.TrimSpace(out) != derivedImage { - c.Fatalf("expected %s; got %s", derivedImage, out) - } - - // Confirm that repushing and repulling does not change the computed ID - dockerCmd(c, "push", derivedImage) - dockerCmd(c, "rmi", derivedImage) - dockerCmd(c, "pull", derivedImage) - - derivedIDAfterPull2 := imageID(c, derivedImage) - derivedParentAfterPull2 := imageParent(c, imageParent(c, derivedImage)) - - if derivedIDAfterPull2 != derivedIDAfterPull1 { - c.Fatal("image's ID unexpectedly changed after a repush/repull") - } - - if derivedParentAfterPull2 != baseIDBeforePush { - c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush) - } - - // Make sure the image still runs - out, _ = dockerCmd(c, "run", "--rm", derivedImage) - if strings.TrimSpace(out) != derivedImage { - c.Fatalf("expected %s; got %s", derivedImage, out) - } -} - -// TestPullConflict provides coverage of the situation where a computed -// strongID conflicts with some unverifiable data in the graph. -func (s *DockerRegistrySuite) TestPullConflict(c *check.C) { - repoName := privateRegistryURL + "/dockercli/conflict" - - _, err := buildImage(repoName, ` - FROM scratch - ENV IMAGE conflict - CMD echo conflict - `, true) - if err != nil { - c.Fatal(err) - } - - dockerCmd(c, "push", repoName) - - // Pull to make it content-addressable - dockerCmd(c, "rmi", repoName) - dockerCmd(c, "pull", repoName) - - IDBeforeLoad := imageID(c, repoName) - - // Load/save to turn this into an unverified image with the same ID - tmpDir, err := ioutil.TempDir("", "conflict-save-output") - if err != nil { - c.Errorf("failed to create temporary directory: %s", err) - } - defer os.RemoveAll(tmpDir) - - tarFile := filepath.Join(tmpDir, "repo.tar") - - dockerCmd(c, "save", "-o", tarFile, repoName) - dockerCmd(c, "rmi", repoName) - dockerCmd(c, "load", "-i", tarFile) - - // Check that the the ID is the same after save/load. - IDAfterLoad := imageID(c, repoName) - - if IDAfterLoad != IDBeforeLoad { - c.Fatal("image's ID should be the same after save/load") - } - - // Repull - dockerCmd(c, "pull", repoName) - - // Check that the ID is now different because of the conflict. - IDAfterPull1 := imageID(c, repoName) - - // Expect the new ID to be SHA256(oldID) - expectedIDDigest, err := digest.FromBytes([]byte(IDBeforeLoad)) - if err != nil { - c.Fatalf("digest error: %v", err) - } - expectedID := expectedIDDigest.Hex() - if IDAfterPull1 != expectedID { - c.Fatalf("image's ID should have changed on pull to %s (got %s)", expectedID, IDAfterPull1) - } - - // A second pull should use the new ID again. - dockerCmd(c, "pull", repoName) - - IDAfterPull2 := imageID(c, repoName) - - if IDAfterPull2 != IDAfterPull1 { - c.Fatal("image's ID unexpectedly changed after a repull") - } -} diff --git a/integration-cli/docker_cli_push_test.go b/integration-cli/docker_cli_push_test.go index 7c4e5c9428f4c..71ebed7d84c83 100644 --- a/integration-cli/docker_cli_push_test.go +++ b/integration-cli/docker_cli_push_test.go @@ -2,16 +2,13 @@ package main import ( "archive/tar" - "encoding/json" "fmt" "io/ioutil" "os" "os/exec" - "path/filepath" "strings" "time" - "github.com/docker/docker/image" "github.com/docker/docker/pkg/integration/checker" "github.com/go-check/check" ) @@ -86,46 +83,6 @@ func (s *DockerRegistrySuite) TestPushMultipleTags(c *check.C) { } } -// TestPushBadParentChain tries to push an image with a corrupted parent chain -// in the v1compatibility files, and makes sure the push process fixes it. -func (s *DockerRegistrySuite) TestPushBadParentChain(c *check.C) { - repoName := fmt.Sprintf("%v/dockercli/badparent", privateRegistryURL) - - id, err := buildImage(repoName, ` - FROM busybox - CMD echo "adding another layer" - `, true) - if err != nil { - c.Fatal(err) - } - - // Push to create v1compatibility file - dockerCmd(c, "push", repoName) - - // Corrupt the parent in the v1compatibility file from the top layer - filename := filepath.Join(dockerBasePath, "graph", id, "v1Compatibility") - - jsonBytes, err := ioutil.ReadFile(filename) - c.Assert(err, check.IsNil, check.Commentf("Could not read v1Compatibility file: %s", err)) - - var img image.Image - err = json.Unmarshal(jsonBytes, &img) - c.Assert(err, check.IsNil, check.Commentf("Could not unmarshal json: %s", err)) - - img.Parent = "1234123412341234123412341234123412341234123412341234123412341234" - - jsonBytes, err = json.Marshal(&img) - c.Assert(err, check.IsNil, check.Commentf("Could not marshal json: %s", err)) - - err = ioutil.WriteFile(filename, jsonBytes, 0600) - c.Assert(err, check.IsNil, check.Commentf("Could not write v1Compatibility file: %s", err)) - - dockerCmd(c, "push", repoName) - - // pull should succeed - dockerCmd(c, "pull", repoName) -} - func (s *DockerRegistrySuite) TestPushEmptyLayer(c *check.C) { repoName := fmt.Sprintf("%v/dockercli/emptylayer", privateRegistryURL) emptyTarball, err := ioutil.TempFile("", "empty_tarball") diff --git a/integration-cli/docker_cli_rmi_test.go b/integration-cli/docker_cli_rmi_test.go index 89bb9db10aff0..13fa08ef64622 100644 --- a/integration-cli/docker_cli_rmi_test.go +++ b/integration-cli/docker_cli_rmi_test.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/docker/docker/pkg/integration/checker" + "github.com/docker/docker/pkg/stringid" "github.com/go-check/check" ) @@ -85,7 +86,7 @@ func (s *DockerSuite) TestRmiImgIDMultipleTag(c *check.C) { // first checkout without force it fails out, _, err = dockerCmdWithError("rmi", imgID) - expected := fmt.Sprintf("conflict: unable to delete %s (cannot be forced) - image is being used by running container %s", imgID[:12], containerID[:12]) + expected := fmt.Sprintf("conflict: unable to delete %s (cannot be forced) - image is being used by running container %s", stringid.TruncateID(imgID), stringid.TruncateID(containerID)) // rmi tagged in multiple repos should have failed without force c.Assert(err, checker.NotNil) c.Assert(out, checker.Contains, expected) diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index 8ec54e5a2bec9..546ad4d5167f2 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -3749,3 +3749,15 @@ func (s *DockerSuite) TestDockerFails(c *check.C) { c.Fatalf("Docker run with flag not defined should exit with 125, but we got out: %s, exit: %d, err: %s", out, exit, err) } } + +// TestRunInvalidReference invokes docker run with a bad reference. +func (s *DockerSuite) TestRunInvalidReference(c *check.C) { + out, exit, _ := dockerCmdWithError("run", "busybox@foo") + if exit == 0 { + c.Fatalf("expected non-zero exist code; received %d", exit) + } + + if !strings.Contains(out, "invalid reference format") { + c.Fatalf(`Expected "invalid reference format" in output; got: %s`, out) + } +} diff --git a/integration-cli/docker_cli_save_load_test.go b/integration-cli/docker_cli_save_load_test.go index 673d3a94a8c99..b81a03319e159 100644 --- a/integration-cli/docker_cli_save_load_test.go +++ b/integration-cli/docker_cli_save_load_test.go @@ -8,10 +8,12 @@ import ( "os/exec" "path/filepath" "reflect" + "regexp" "sort" "strings" "time" + "github.com/docker/distribution/digest" "github.com/docker/docker/pkg/integration/checker" "github.com/go-check/check" ) @@ -100,7 +102,7 @@ func (s *DockerSuite) TestSaveCheckTimes(c *check.C) { out, _, err = runCommandPipelineWithOutput( exec.Command(dockerBinary, "save", repoName), exec.Command("tar", "tv"), - exec.Command("grep", "-E", fmt.Sprintf("%s %s", data[0].Created.Format(tarTvTimeFormat), data[0].ID))) + exec.Command("grep", "-E", fmt.Sprintf("%s %s", data[0].Created.Format(tarTvTimeFormat), digest.Digest(data[0].ID).Hex()))) c.Assert(err, checker.IsNil, check.Commentf("failed to save repo with image ID and 'repositories' file: %s, %v", out, err)) } @@ -110,7 +112,7 @@ func (s *DockerSuite) TestSaveImageId(c *check.C) { dockerCmd(c, "tag", "emptyfs:latest", fmt.Sprintf("%v:latest", repoName)) out, _ := dockerCmd(c, "images", "-q", "--no-trunc", repoName) - cleanedLongImageID := strings.TrimSpace(out) + cleanedLongImageID := strings.TrimPrefix(strings.TrimSpace(out), "sha256:") out, _ = dockerCmd(c, "images", "-q", repoName) cleanedShortImageID := strings.TrimSpace(out) @@ -207,20 +209,30 @@ func (s *DockerSuite) TestSaveRepoWithMultipleImages(c *check.C) { // create the archive out, _, err := runCommandPipelineWithOutput( - exec.Command(dockerBinary, "save", repoName), - exec.Command("tar", "t"), - exec.Command("grep", "VERSION"), - exec.Command("cut", "-d", "/", "-f1")) + exec.Command(dockerBinary, "save", repoName, "busybox:latest"), + exec.Command("tar", "t")) c.Assert(err, checker.IsNil, check.Commentf("failed to save multiple images: %s, %v", out, err)) - actual := strings.Split(strings.TrimSpace(out), "\n") + + lines := strings.Split(strings.TrimSpace(out), "\n") + var actual []string + for _, l := range lines { + if regexp.MustCompile("^[a-f0-9]{64}\\.json$").Match([]byte(l)) { + actual = append(actual, strings.TrimSuffix(l, ".json")) + } + } // make the list of expected layers - out, _ = dockerCmd(c, "history", "-q", "--no-trunc", "busybox:latest") - expected := append(strings.Split(strings.TrimSpace(out), "\n"), idFoo, idBar) + out, _ = dockerCmd(c, "inspect", "-f", "{{.Id}}", "busybox:latest") + expected := []string{strings.TrimSpace(out), idFoo, idBar} + + // prefixes are not in tar + for i := range expected { + expected[i] = digest.Digest(expected[i]).Hex() + } sort.Strings(actual) sort.Strings(expected) - c.Assert(actual, checker.DeepEquals, expected, check.Commentf("archive does not contains the right layers: got %v, expected %v", actual, expected)) + c.Assert(actual, checker.DeepEquals, expected, check.Commentf("archive does not contains the right layers: got %v, expected %v, output: %q", actual, expected, out)) } // Issue #6722 #5892 ensure directories are included in changes diff --git a/integration-cli/docker_cli_save_load_unix_test.go b/integration-cli/docker_cli_save_load_unix_test.go index 185c1a716216c..cef7d43079848 100644 --- a/integration-cli/docker_cli_save_load_unix_test.go +++ b/integration-cli/docker_cli_save_load_unix_test.go @@ -18,9 +18,7 @@ func (s *DockerSuite) TestSaveAndLoadRepoStdout(c *check.C) { dockerCmd(c, "run", "--name", name, "busybox", "true") repoName := "foobar-save-load-test" - out, _ := dockerCmd(c, "commit", name, repoName) - - before, _ := dockerCmd(c, "inspect", repoName) + before, _ := dockerCmd(c, "commit", name, repoName) tmpFile, err := ioutil.TempFile("", "foobar-save-load-test.tar") c.Assert(err, check.IsNil) @@ -40,10 +38,10 @@ func (s *DockerSuite) TestSaveAndLoadRepoStdout(c *check.C) { loadCmd := exec.Command(dockerBinary, "load") loadCmd.Stdin = tmpFile - out, _, err = runCommandWithOutput(loadCmd) + out, _, err := runCommandWithOutput(loadCmd) c.Assert(err, check.IsNil, check.Commentf(out)) - after, _ := dockerCmd(c, "inspect", repoName) + after, _ := dockerCmd(c, "inspect", "-f", "{{.Id}}", repoName) c.Assert(before, check.Equals, after) //inspect is not the same after a save / load diff --git a/integration-cli/docker_cli_tag_test.go b/integration-cli/docker_cli_tag_test.go index cd0ca4126d4f2..aa812174923ae 100644 --- a/integration-cli/docker_cli_tag_test.go +++ b/integration-cli/docker_cli_tag_test.go @@ -1,9 +1,11 @@ package main import ( + "fmt" "strings" "github.com/docker/docker/pkg/integration/checker" + "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/stringutils" "github.com/go-check/check" ) @@ -111,7 +113,7 @@ func (s *DockerSuite) TestTagWithPrefixHyphen(c *check.C) { // test index name begin with '-' out, _, err = dockerCmdWithError("tag", "busybox:latest", "-index:5000/busybox:test") c.Assert(err, checker.NotNil, check.Commentf(out)) - c.Assert(out, checker.Contains, "Invalid index name (-index:5000). Cannot begin or end with a hyphen", check.Commentf("tag a name begin with '-' should failed")) + c.Assert(out, checker.Contains, "invalid reference format", check.Commentf("tag a name begin with '-' should failed")) } // ensure tagging using official names works @@ -171,3 +173,57 @@ func (s *DockerSuite) TestTagMatchesDigest(c *check.C) { c.Fatal("inspecting by digest should have failed") } } + +func (s *DockerSuite) TestTagInvalidRepoName(c *check.C) { + testRequires(c, DaemonIsLinux) + if err := pullImageIfNotExist("busybox:latest"); err != nil { + c.Fatal("couldn't find the busybox:latest image locally and failed to pull it") + } + + // test setting tag fails + _, _, err := dockerCmdWithError("tag", "-f", "busybox:latest", "sha256:sometag") + if err == nil { + c.Fatal("tagging with image named \"sha256\" should have failed") + } +} + +// ensure tags cannot create ambiguity with image ids +func (s *DockerSuite) TestTagTruncationAmbiguity(c *check.C) { + testRequires(c, DaemonIsLinux) + if err := pullImageIfNotExist("busybox:latest"); err != nil { + c.Fatal("couldn't find the busybox:latest image locally and failed to pull it") + } + + imageID, err := buildImage("notbusybox:latest", + `FROM busybox + MAINTAINER dockerio`, + true) + if err != nil { + c.Fatal(err) + } + truncatedImageID := stringid.TruncateID(imageID) + truncatedTag := fmt.Sprintf("notbusybox:%s", truncatedImageID) + + id, err := inspectField(truncatedTag, "Id") + if err != nil { + c.Fatalf("Error inspecting by image id: %s", err) + } + + // Ensure inspect by image id returns image for image id + c.Assert(id, checker.Equals, imageID) + c.Logf("Built image: %s", imageID) + + // test setting tag fails + _, _, err = dockerCmdWithError("tag", "-f", "busybox:latest", truncatedTag) + if err != nil { + c.Fatalf("Error tagging with an image id: %s", err) + } + + id, err = inspectField(truncatedTag, "Id") + if err != nil { + c.Fatalf("Error inspecting by image id: %s", err) + } + + // Ensure id is imageID and not busybox:latest + c.Assert(id, checker.Not(checker.Equals), imageID) +} diff --git a/pkg/parsers/parsers.go b/pkg/parsers/parsers.go index 59fea5cd9e9f7..51de847b16061 100644 --- a/pkg/parsers/parsers.go +++ b/pkg/parsers/parsers.go @@ -110,26 +110,6 @@ func ParseTCPAddr(tryAddr string, defaultAddr string) (string, error) { return fmt.Sprintf("tcp://%s%s", net.JoinHostPort(host, port), u.Path), nil } -// ParseRepositoryTag gets a repos name and returns the right reposName + tag|digest -// The tag can be confusing because of a port in a repository name. -// Ex: localhost.localdomain:5000/samalba/hipache:latest -// Digest ex: localhost:5000/foo/bar@sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb -func ParseRepositoryTag(repos string) (string, string) { - n := strings.Index(repos, "@") - if n >= 0 { - parts := strings.Split(repos, "@") - return parts[0], parts[1] - } - n = strings.LastIndex(repos, ":") - if n < 0 { - return repos, "" - } - if tag := repos[n+1:]; !strings.Contains(tag, "/") { - return repos[:n], tag - } - return repos, "" -} - // PartParser parses and validates the specified string (data) using the specified template // e.g. ip:public:private -> 192.168.0.1:80:8000 func PartParser(template, data string) (map[string]string, error) { diff --git a/pkg/parsers/parsers_test.go b/pkg/parsers/parsers_test.go index 47b45281c790c..db538d583e449 100644 --- a/pkg/parsers/parsers_test.go +++ b/pkg/parsers/parsers_test.go @@ -120,36 +120,6 @@ func TestParseInvalidUnixAddrInvalid(t *testing.T) { } } -func TestParseRepositoryTag(t *testing.T) { - if repo, tag := ParseRepositoryTag("root"); repo != "root" || tag != "" { - t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "", repo, tag) - } - if repo, tag := ParseRepositoryTag("root:tag"); repo != "root" || tag != "tag" { - t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "tag", repo, tag) - } - if repo, digest := ParseRepositoryTag("root@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); repo != "root" || digest != "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { - t.Errorf("Expected repo: '%s' and digest: '%s', got '%s' and '%s'", "root", "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", repo, digest) - } - if repo, tag := ParseRepositoryTag("user/repo"); repo != "user/repo" || tag != "" { - t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "", repo, tag) - } - if repo, tag := ParseRepositoryTag("user/repo:tag"); repo != "user/repo" || tag != "tag" { - t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "tag", repo, tag) - } - if repo, digest := ParseRepositoryTag("user/repo@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); repo != "user/repo" || digest != "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { - t.Errorf("Expected repo: '%s' and digest: '%s', got '%s' and '%s'", "user/repo", "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", repo, digest) - } - if repo, tag := ParseRepositoryTag("url:5000/repo"); repo != "url:5000/repo" || tag != "" { - t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "", repo, tag) - } - if repo, tag := ParseRepositoryTag("url:5000/repo:tag"); repo != "url:5000/repo" || tag != "tag" { - t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "tag", repo, tag) - } - if repo, digest := ParseRepositoryTag("url:5000/repo@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); repo != "url:5000/repo" || digest != "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { - t.Errorf("Expected repo: '%s' and digest: '%s', got '%s' and '%s'", "url:5000/repo", "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", repo, digest) - } -} - func TestParseKeyValueOpt(t *testing.T) { invalids := map[string]string{ "": "Unable to parse key/value option: ", diff --git a/pkg/stringid/stringid.go b/pkg/stringid/stringid.go index 266a74ba6dd2f..0332da6676c36 100644 --- a/pkg/stringid/stringid.go +++ b/pkg/stringid/stringid.go @@ -7,6 +7,7 @@ import ( "io" "regexp" "strconv" + "strings" "github.com/docker/docker/pkg/random" ) @@ -25,6 +26,9 @@ func IsShortID(id string) bool { // In case of a collision a lookup with TruncIndex.Get() will fail, and the caller // will need to use a langer prefix, or the full-length Id. func TruncateID(id string) string { + if i := strings.IndexRune(id, ':'); i >= 0 { + id = id[i+1:] + } trimTo := shortLen if len(id) < shortLen { trimTo = len(id) diff --git a/registry/config.go b/registry/config.go index 7cac7158400e1..8d7962f8d4b64 100644 --- a/registry/config.go +++ b/registry/config.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/docker/distribution/reference" - "github.com/docker/docker/image" + "github.com/docker/docker/image/v1" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" ) @@ -216,18 +216,15 @@ func ValidateIndexName(val string) (string, error) { return val, nil } -func validateRemoteName(remoteName string) error { - - if !strings.Contains(remoteName, "/") { - +func validateRemoteName(remoteName reference.Named) error { + remoteNameStr := remoteName.Name() + if !strings.Contains(remoteNameStr, "/") { // the repository name must not be a valid image ID - if err := image.ValidateID(remoteName); err == nil { + if err := v1.ValidateID(remoteNameStr); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", remoteName) } } - - _, err := reference.WithName(remoteName) - return err + return nil } func validateNoSchema(reposName string) error { @@ -239,27 +236,24 @@ func validateNoSchema(reposName string) error { } // ValidateRepositoryName validates a repository name -func ValidateRepositoryName(reposName string) error { - _, _, err := loadRepositoryName(reposName, true) +func ValidateRepositoryName(reposName reference.Named) error { + _, _, err := loadRepositoryName(reposName) return err } // loadRepositoryName returns the repo name splitted into index name // and remote repo name. It returns an error if the name is not valid. -func loadRepositoryName(reposName string, checkRemoteName bool) (string, string, error) { - if err := validateNoSchema(reposName); err != nil { - return "", "", err +func loadRepositoryName(reposName reference.Named) (string, reference.Named, error) { + if err := validateNoSchema(reposName.Name()); err != nil { + return "", nil, err } - indexName, remoteName := splitReposName(reposName) + indexName, remoteName, err := splitReposName(reposName) - var err error if indexName, err = ValidateIndexName(indexName); err != nil { - return "", "", err + return "", nil, err } - if checkRemoteName { - if err = validateRemoteName(remoteName); err != nil { - return "", "", err - } + if err = validateRemoteName(remoteName); err != nil { + return "", nil, err } return indexName, remoteName, nil } @@ -297,31 +291,36 @@ func (index *IndexInfo) GetAuthConfigKey() string { } // splitReposName breaks a reposName into an index name and remote name -func splitReposName(reposName string) (string, string) { - nameParts := strings.SplitN(reposName, "/", 2) - var indexName, remoteName string - if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && - !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { +func splitReposName(reposName reference.Named) (indexName string, remoteName reference.Named, err error) { + var remoteNameStr string + indexName, remoteNameStr = reference.SplitHostname(reposName) + if indexName == "" || (!strings.Contains(indexName, ".") && + !strings.Contains(indexName, ":") && indexName != "localhost") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) // 'docker.io' indexName = IndexName remoteName = reposName } else { - indexName = nameParts[0] - remoteName = nameParts[1] + remoteName, err = reference.WithName(remoteNameStr) } - return indexName, remoteName + return } // NewRepositoryInfo validates and breaks down a repository name into a RepositoryInfo -func (config *ServiceConfig) NewRepositoryInfo(reposName string, bySearch bool) (*RepositoryInfo, error) { - indexName, remoteName, err := loadRepositoryName(reposName, !bySearch) - if err != nil { +func (config *ServiceConfig) NewRepositoryInfo(reposName reference.Named) (*RepositoryInfo, error) { + if err := validateNoSchema(reposName.Name()); err != nil { return nil, err } - repoInfo := &RepositoryInfo{ - RemoteName: remoteName, + repoInfo := &RepositoryInfo{} + var ( + indexName string + err error + ) + + indexName, repoInfo.RemoteName, err = loadRepositoryName(reposName) + if err != nil { + return nil, err } repoInfo.Index, err = config.NewIndexInfo(indexName) @@ -330,46 +329,47 @@ func (config *ServiceConfig) NewRepositoryInfo(reposName string, bySearch bool) } if repoInfo.Index.Official { - normalizedName := normalizeLibraryRepoName(repoInfo.RemoteName) + repoInfo.LocalName, err = normalizeLibraryRepoName(repoInfo.RemoteName) + if err != nil { + return nil, err + } + repoInfo.RemoteName = repoInfo.LocalName - repoInfo.LocalName = normalizedName - repoInfo.RemoteName = normalizedName // If the normalized name does not contain a '/' (e.g. "foo") // then it is an official repo. - if strings.IndexRune(normalizedName, '/') == -1 { + if strings.IndexRune(repoInfo.RemoteName.Name(), '/') == -1 { repoInfo.Official = true // Fix up remote name for official repos. - repoInfo.RemoteName = "library/" + normalizedName + repoInfo.RemoteName, err = reference.WithName("library/" + repoInfo.RemoteName.Name()) + if err != nil { + return nil, err + } } - repoInfo.CanonicalName = "docker.io/" + repoInfo.RemoteName + repoInfo.CanonicalName, err = reference.WithName("docker.io/" + repoInfo.RemoteName.Name()) + if err != nil { + return nil, err + } } else { - repoInfo.LocalName = localNameFromRemote(repoInfo.Index.Name, repoInfo.RemoteName) + repoInfo.LocalName, err = localNameFromRemote(repoInfo.Index.Name, repoInfo.RemoteName) + if err != nil { + return nil, err + } repoInfo.CanonicalName = repoInfo.LocalName - } return repoInfo, nil } -// GetSearchTerm special-cases using local name for official index, and -// remote name for private indexes. -func (repoInfo *RepositoryInfo) GetSearchTerm() string { - if repoInfo.Index.Official { - return repoInfo.LocalName - } - return repoInfo.RemoteName -} - // ParseRepositoryInfo performs the breakdown of a repository name into a RepositoryInfo, but // lacks registry configuration. -func ParseRepositoryInfo(reposName string) (*RepositoryInfo, error) { - return emptyServiceConfig.NewRepositoryInfo(reposName, false) +func ParseRepositoryInfo(reposName reference.Named) (*RepositoryInfo, error) { + return emptyServiceConfig.NewRepositoryInfo(reposName) } -// ParseIndexInfo will use repository name to get back an indexInfo. -func ParseIndexInfo(reposName string) (*IndexInfo, error) { - indexName, _ := splitReposName(reposName) +// ParseSearchIndexInfo will use repository name to get back an indexInfo. +func ParseSearchIndexInfo(reposName string) (*IndexInfo, error) { + indexName, _ := splitReposSearchTerm(reposName) indexInfo, err := emptyServiceConfig.NewIndexInfo(indexName) if err != nil { @@ -378,12 +378,12 @@ func ParseIndexInfo(reposName string) (*IndexInfo, error) { return indexInfo, nil } -// NormalizeLocalName transforms a repository name into a normalize LocalName +// NormalizeLocalName transforms a repository name into a normalized LocalName // Passes through the name without transformation on error (image id, etc) // It does not use the repository info because we don't want to load // the repository index and do request over the network. -func NormalizeLocalName(name string) string { - indexName, remoteName, err := loadRepositoryName(name, true) +func NormalizeLocalName(name reference.Named) reference.Named { + indexName, remoteName, err := loadRepositoryName(name) if err != nil { return name } @@ -395,23 +395,52 @@ func NormalizeLocalName(name string) string { } if officialIndex { - return normalizeLibraryRepoName(remoteName) + localName, err := normalizeLibraryRepoName(remoteName) + if err != nil { + return name + } + return localName } - return localNameFromRemote(indexName, remoteName) + localName, err := localNameFromRemote(indexName, remoteName) + if err != nil { + return name + } + return localName } // normalizeLibraryRepoName removes the library prefix from // the repository name for official repos. -func normalizeLibraryRepoName(name string) string { - if strings.HasPrefix(name, "library/") { +func normalizeLibraryRepoName(name reference.Named) (reference.Named, error) { + if strings.HasPrefix(name.Name(), "library/") { // If pull "library/foo", it's stored locally under "foo" - name = strings.SplitN(name, "/", 2)[1] + return reference.WithName(strings.SplitN(name.Name(), "/", 2)[1]) } - return name + return name, nil } // localNameFromRemote combines the index name and the repo remote name // to generate a repo local name. -func localNameFromRemote(indexName, remoteName string) string { - return indexName + "/" + remoteName +func localNameFromRemote(indexName string, remoteName reference.Named) (reference.Named, error) { + return reference.WithName(indexName + "/" + remoteName.Name()) +} + +// NormalizeLocalReference transforms a reference to use a normalized LocalName +// for the name poriton. Passes through the reference without transformation on +// error. +func NormalizeLocalReference(ref reference.Named) reference.Named { + localName := NormalizeLocalName(ref) + if tagged, isTagged := ref.(reference.Tagged); isTagged { + newRef, err := reference.WithTag(localName, tagged.Tag()) + if err != nil { + return ref + } + return newRef + } else if digested, isDigested := ref.(reference.Digested); isDigested { + newRef, err := reference.WithDigest(localName, digested.Digest()) + if err != nil { + return ref + } + return newRef + } + return localName } diff --git a/registry/registry_mock_test.go b/registry/registry_mock_test.go index fb19e577dd170..3c75dea6d89d8 100644 --- a/registry/registry_mock_test.go +++ b/registry/registry_mock_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/docker/distribution/reference" "github.com/docker/docker/opts" "github.com/gorilla/mux" @@ -349,15 +350,19 @@ func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { if !requiresAuth(w, r) { return } - repositoryName := mux.Vars(r)["repository"] + repositoryName, err := reference.WithName(mux.Vars(r)["repository"]) + if err != nil { + apiError(w, "Could not parse repository", 400) + return + } repositoryName = NormalizeLocalName(repositoryName) - tags, exists := testRepositories[repositoryName] + tags, exists := testRepositories[repositoryName.String()] if !exists { apiError(w, "Repository not found", 404) return } if r.Method == "DELETE" { - delete(testRepositories, repositoryName) + delete(testRepositories, repositoryName.String()) writeResponse(w, true, 200) return } @@ -369,10 +374,14 @@ func handlerGetTag(w http.ResponseWriter, r *http.Request) { return } vars := mux.Vars(r) - repositoryName := vars["repository"] + repositoryName, err := reference.WithName(vars["repository"]) + if err != nil { + apiError(w, "Could not parse repository", 400) + return + } repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] - tags, exists := testRepositories[repositoryName] + tags, exists := testRepositories[repositoryName.String()] if !exists { apiError(w, "Repository not found", 404) return @@ -390,13 +399,17 @@ func handlerPutTag(w http.ResponseWriter, r *http.Request) { return } vars := mux.Vars(r) - repositoryName := vars["repository"] + repositoryName, err := reference.WithName(vars["repository"]) + if err != nil { + apiError(w, "Could not parse repository", 400) + return + } repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] - tags, exists := testRepositories[repositoryName] + tags, exists := testRepositories[repositoryName.String()] if !exists { - tags := make(map[string]string) - testRepositories[repositoryName] = tags + tags = make(map[string]string) + testRepositories[repositoryName.String()] = tags } tagValue := "" readJSON(r, tagValue) diff --git a/registry/registry_test.go b/registry/registry_test.go index 7714310d93389..2bc1edff7302b 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/cliconfig" ) @@ -214,13 +215,21 @@ func TestGetRemoteImageLayer(t *testing.T) { func TestGetRemoteTag(t *testing.T) { r := spawnTestRegistrySession(t) - tag, err := r.GetRemoteTag([]string{makeURL("/v1/")}, REPO, "test") + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + tag, err := r.GetRemoteTag([]string{makeURL("/v1/")}, repoRef, "test") if err != nil { t.Fatal(err) } assertEqual(t, tag, imageID, "Expected tag test to map to "+imageID) - _, err = r.GetRemoteTag([]string{makeURL("/v1/")}, "foo42/baz", "foo") + bazRef, err := reference.ParseNamed("foo42/baz") + if err != nil { + t.Fatal(err) + } + _, err = r.GetRemoteTag([]string{makeURL("/v1/")}, bazRef, "foo") if err != ErrRepoNotFound { t.Fatal("Expected ErrRepoNotFound error when fetching tag for bogus repo") } @@ -228,7 +237,11 @@ func TestGetRemoteTag(t *testing.T) { func TestGetRemoteTags(t *testing.T) { r := spawnTestRegistrySession(t) - tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO) + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, repoRef) if err != nil { t.Fatal(err) } @@ -236,7 +249,11 @@ func TestGetRemoteTags(t *testing.T) { assertEqual(t, tags["latest"], imageID, "Expected tag latest to map to "+imageID) assertEqual(t, tags["test"], imageID, "Expected tag test to map to "+imageID) - _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz") + bazRef, err := reference.ParseNamed("foo42/baz") + if err != nil { + t.Fatal(err) + } + _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, bazRef) if err != ErrRepoNotFound { t.Fatal("Expected ErrRepoNotFound error when fetching tags for bogus repo") } @@ -249,7 +266,11 @@ func TestGetRepositoryData(t *testing.T) { t.Fatal(err) } host := "http://" + parsedURL.Host + "/v1/" - data, err := r.GetRepositoryData("foo42/bar") + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + data, err := r.GetRepositoryData(repoRef) if err != nil { t.Fatal(err) } @@ -315,29 +336,41 @@ func TestValidateRepositoryName(t *testing.T) { } for _, name := range invalidRepoNames { - err := ValidateRepositoryName(name) - assertNotEqual(t, err, nil, "Expected invalid repo name: "+name) + named, err := reference.WithName(name) + if err == nil { + err := ValidateRepositoryName(named) + assertNotEqual(t, err, nil, "Expected invalid repo name: "+name) + } } for _, name := range validRepoNames { - err := ValidateRepositoryName(name) + named, err := reference.WithName(name) + if err != nil { + t.Fatalf("could not parse valid name: %s", name) + } + err = ValidateRepositoryName(named) assertEqual(t, err, nil, "Expected valid repo name: "+name) } - - err := ValidateRepositoryName(invalidRepoNames[0]) - assertEqual(t, err, ErrInvalidRepositoryName, "Expected ErrInvalidRepositoryName: "+invalidRepoNames[0]) } func TestParseRepositoryInfo(t *testing.T) { + withName := func(name string) reference.Named { + named, err := reference.WithName(name) + if err != nil { + t.Fatalf("could not parse reference %s", name) + } + return named + } + expectedRepoInfos := map[string]RepositoryInfo{ "fooo/bar": { Index: &IndexInfo{ Name: IndexName, Official: true, }, - RemoteName: "fooo/bar", - LocalName: "fooo/bar", - CanonicalName: "docker.io/fooo/bar", + RemoteName: withName("fooo/bar"), + LocalName: withName("fooo/bar"), + CanonicalName: withName("docker.io/fooo/bar"), Official: false, }, "library/ubuntu": { @@ -345,9 +378,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu", - LocalName: "ubuntu", - CanonicalName: "docker.io/library/ubuntu", + RemoteName: withName("library/ubuntu"), + LocalName: withName("ubuntu"), + CanonicalName: withName("docker.io/library/ubuntu"), Official: true, }, "nonlibrary/ubuntu": { @@ -355,9 +388,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "nonlibrary/ubuntu", - LocalName: "nonlibrary/ubuntu", - CanonicalName: "docker.io/nonlibrary/ubuntu", + RemoteName: withName("nonlibrary/ubuntu"), + LocalName: withName("nonlibrary/ubuntu"), + CanonicalName: withName("docker.io/nonlibrary/ubuntu"), Official: false, }, "ubuntu": { @@ -365,9 +398,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu", - LocalName: "ubuntu", - CanonicalName: "docker.io/library/ubuntu", + RemoteName: withName("library/ubuntu"), + LocalName: withName("ubuntu"), + CanonicalName: withName("docker.io/library/ubuntu"), Official: true, }, "other/library": { @@ -375,9 +408,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "other/library", - LocalName: "other/library", - CanonicalName: "docker.io/other/library", + RemoteName: withName("other/library"), + LocalName: withName("other/library"), + CanonicalName: withName("docker.io/other/library"), Official: false, }, "127.0.0.1:8000/private/moonbase": { @@ -385,9 +418,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "127.0.0.1:8000", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "127.0.0.1:8000/private/moonbase", - CanonicalName: "127.0.0.1:8000/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("127.0.0.1:8000/private/moonbase"), + CanonicalName: withName("127.0.0.1:8000/private/moonbase"), Official: false, }, "127.0.0.1:8000/privatebase": { @@ -395,9 +428,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "127.0.0.1:8000", Official: false, }, - RemoteName: "privatebase", - LocalName: "127.0.0.1:8000/privatebase", - CanonicalName: "127.0.0.1:8000/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("127.0.0.1:8000/privatebase"), + CanonicalName: withName("127.0.0.1:8000/privatebase"), Official: false, }, "localhost:8000/private/moonbase": { @@ -405,9 +438,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost:8000", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "localhost:8000/private/moonbase", - CanonicalName: "localhost:8000/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("localhost:8000/private/moonbase"), + CanonicalName: withName("localhost:8000/private/moonbase"), Official: false, }, "localhost:8000/privatebase": { @@ -415,9 +448,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost:8000", Official: false, }, - RemoteName: "privatebase", - LocalName: "localhost:8000/privatebase", - CanonicalName: "localhost:8000/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("localhost:8000/privatebase"), + CanonicalName: withName("localhost:8000/privatebase"), Official: false, }, "example.com/private/moonbase": { @@ -425,9 +458,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "example.com/private/moonbase", - CanonicalName: "example.com/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("example.com/private/moonbase"), + CanonicalName: withName("example.com/private/moonbase"), Official: false, }, "example.com/privatebase": { @@ -435,9 +468,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com", Official: false, }, - RemoteName: "privatebase", - LocalName: "example.com/privatebase", - CanonicalName: "example.com/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("example.com/privatebase"), + CanonicalName: withName("example.com/privatebase"), Official: false, }, "example.com:8000/private/moonbase": { @@ -445,9 +478,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com:8000", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "example.com:8000/private/moonbase", - CanonicalName: "example.com:8000/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("example.com:8000/private/moonbase"), + CanonicalName: withName("example.com:8000/private/moonbase"), Official: false, }, "example.com:8000/privatebase": { @@ -455,9 +488,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com:8000", Official: false, }, - RemoteName: "privatebase", - LocalName: "example.com:8000/privatebase", - CanonicalName: "example.com:8000/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("example.com:8000/privatebase"), + CanonicalName: withName("example.com:8000/privatebase"), Official: false, }, "localhost/private/moonbase": { @@ -465,9 +498,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "localhost/private/moonbase", - CanonicalName: "localhost/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("localhost/private/moonbase"), + CanonicalName: withName("localhost/private/moonbase"), Official: false, }, "localhost/privatebase": { @@ -475,9 +508,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost", Official: false, }, - RemoteName: "privatebase", - LocalName: "localhost/privatebase", - CanonicalName: "localhost/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("localhost/privatebase"), + CanonicalName: withName("localhost/privatebase"), Official: false, }, IndexName + "/public/moonbase": { @@ -485,9 +518,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "public/moonbase", - LocalName: "public/moonbase", - CanonicalName: "docker.io/public/moonbase", + RemoteName: withName("public/moonbase"), + LocalName: withName("public/moonbase"), + CanonicalName: withName("docker.io/public/moonbase"), Official: false, }, "index." + IndexName + "/public/moonbase": { @@ -495,9 +528,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "public/moonbase", - LocalName: "public/moonbase", - CanonicalName: "docker.io/public/moonbase", + RemoteName: withName("public/moonbase"), + LocalName: withName("public/moonbase"), + CanonicalName: withName("docker.io/public/moonbase"), Official: false, }, "ubuntu-12.04-base": { @@ -505,9 +538,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu-12.04-base", - LocalName: "ubuntu-12.04-base", - CanonicalName: "docker.io/library/ubuntu-12.04-base", + RemoteName: withName("library/ubuntu-12.04-base"), + LocalName: withName("ubuntu-12.04-base"), + CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), Official: true, }, IndexName + "/ubuntu-12.04-base": { @@ -515,9 +548,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu-12.04-base", - LocalName: "ubuntu-12.04-base", - CanonicalName: "docker.io/library/ubuntu-12.04-base", + RemoteName: withName("library/ubuntu-12.04-base"), + LocalName: withName("ubuntu-12.04-base"), + CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), Official: true, }, "index." + IndexName + "/ubuntu-12.04-base": { @@ -525,22 +558,27 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu-12.04-base", - LocalName: "ubuntu-12.04-base", - CanonicalName: "docker.io/library/ubuntu-12.04-base", + RemoteName: withName("library/ubuntu-12.04-base"), + LocalName: withName("ubuntu-12.04-base"), + CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), Official: true, }, } for reposName, expectedRepoInfo := range expectedRepoInfos { - repoInfo, err := ParseRepositoryInfo(reposName) + named, err := reference.WithName(reposName) + if err != nil { + t.Error(err) + } + + repoInfo, err := ParseRepositoryInfo(named) if err != nil { t.Error(err) } else { checkEqual(t, repoInfo.Index.Name, expectedRepoInfo.Index.Name, reposName) - checkEqual(t, repoInfo.RemoteName, expectedRepoInfo.RemoteName, reposName) - checkEqual(t, repoInfo.LocalName, expectedRepoInfo.LocalName, reposName) - checkEqual(t, repoInfo.CanonicalName, expectedRepoInfo.CanonicalName, reposName) + checkEqual(t, repoInfo.RemoteName.String(), expectedRepoInfo.RemoteName.String(), reposName) + checkEqual(t, repoInfo.LocalName.String(), expectedRepoInfo.LocalName.String(), reposName) + checkEqual(t, repoInfo.CanonicalName.String(), expectedRepoInfo.CanonicalName.String(), reposName) checkEqual(t, repoInfo.Index.Official, expectedRepoInfo.Index.Official, reposName) checkEqual(t, repoInfo.Official, expectedRepoInfo.Official, reposName) } @@ -687,8 +725,11 @@ func TestMirrorEndpointLookup(t *testing.T) { return false } s := Service{Config: makeServiceConfig([]string{"my.mirror"}, nil)} - imageName := IndexName + "/test/image" + imageName, err := reference.WithName(IndexName + "/test/image") + if err != nil { + t.Error(err) + } pushAPIEndpoints, err := s.LookupPushEndpoints(imageName) if err != nil { t.Fatal(err) @@ -708,7 +749,11 @@ func TestMirrorEndpointLookup(t *testing.T) { func TestPushRegistryTag(t *testing.T) { r := spawnTestRegistrySession(t) - err := r.PushRegistryTag("foo42/bar", imageID, "stable", makeURL("/v1/")) + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + err = r.PushRegistryTag(repoRef, imageID, "stable", makeURL("/v1/")) if err != nil { t.Fatal(err) } @@ -726,14 +771,18 @@ func TestPushImageJSONIndex(t *testing.T) { Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", }, } - repoData, err := r.PushImageJSONIndex("foo42/bar", imgData, false, nil) + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + repoData, err := r.PushImageJSONIndex(repoRef, imgData, false, nil) if err != nil { t.Fatal(err) } if repoData == nil { t.Fatal("Expected RepositoryData object") } - repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint.String()}) + repoData, err = r.PushImageJSONIndex(repoRef, imgData, true, []string{r.indexEndpoint.String()}) if err != nil { t.Fatal(err) } @@ -781,7 +830,11 @@ func TestValidRemoteName(t *testing.T) { "dock__er/docker", } for _, repositoryName := range validRepositoryNames { - if err := validateRemoteName(repositoryName); err != nil { + repositoryRef, err := reference.WithName(repositoryName) + if err != nil { + t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) + } + if err := validateRemoteName(repositoryRef); err != nil { t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) } } @@ -818,7 +871,11 @@ func TestValidRemoteName(t *testing.T) { "this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255/docker", } for _, repositoryName := range invalidRepositoryNames { - if err := validateRemoteName(repositoryName); err == nil { + repositoryRef, err := reference.ParseNamed(repositoryName) + if err != nil { + continue + } + if err := validateRemoteName(repositoryRef); err == nil { t.Errorf("Repository name should be invalid: %v", repositoryName) } } diff --git a/registry/service.go b/registry/service.go index 6ac930d6e358a..1ef9682785ee3 100644 --- a/registry/service.go +++ b/registry/service.go @@ -4,7 +4,9 @@ import ( "crypto/tls" "net/http" "net/url" + "strings" + "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/cliconfig" ) @@ -51,17 +53,39 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { return Login(authConfig, endpoint) } +// splitReposSearchTerm breaks a search term into an index name and remote name +func splitReposSearchTerm(reposName string) (string, string) { + nameParts := strings.SplitN(reposName, "/", 2) + var indexName, remoteName string + if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && + !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { + // This is a Docker Index repos (ex: samalba/hipache or ubuntu) + // 'docker.io' + indexName = IndexName + remoteName = reposName + } else { + indexName = nameParts[0] + remoteName = nameParts[1] + } + return indexName, remoteName +} + // Search queries the public registry for images matching the specified // search terms, and returns the results. func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers map[string][]string) (*SearchResults, error) { + if err := validateNoSchema(term); err != nil { + return nil, err + } + + indexName, remoteName := splitReposSearchTerm(term) - repoInfo, err := s.ResolveRepositoryBySearch(term) + index, err := s.Config.NewIndexInfo(indexName) if err != nil { return nil, err } // *TODO: Search multiple indexes. - endpoint, err := NewEndpoint(repoInfo.Index, http.Header(headers), APIVersionUnknown) + endpoint, err := NewEndpoint(index, http.Header(headers), APIVersionUnknown) if err != nil { return nil, err } @@ -70,19 +94,23 @@ func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers if err != nil { return nil, err } - return r.SearchRepositories(repoInfo.GetSearchTerm()) -} -// ResolveRepository splits a repository name into its components -// and configuration of the associated registry. -func (s *Service) ResolveRepository(name string) (*RepositoryInfo, error) { - return s.Config.NewRepositoryInfo(name, false) + if index.Official { + localName := remoteName + if strings.HasPrefix(localName, "library/") { + // If pull "library/foo", it's stored locally under "foo" + localName = strings.SplitN(localName, "/", 2)[1] + } + + return r.SearchRepositories(localName) + } + return r.SearchRepositories(remoteName) } -// ResolveRepositoryBySearch splits a repository name into its components +// ResolveRepository splits a repository name into its components // and configuration of the associated registry. -func (s *Service) ResolveRepositoryBySearch(name string) (*RepositoryInfo, error) { - return s.Config.NewRepositoryInfo(name, true) +func (s *Service) ResolveRepository(name reference.Named) (*RepositoryInfo, error) { + return s.Config.NewRepositoryInfo(name) } // ResolveIndex takes indexName and returns index info @@ -123,14 +151,14 @@ func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { // LookupPullEndpoints creates an list of endpoints to try to pull from, in order of preference. // It gives preference to v2 endpoints over v1, mirrors over the actual // registry, and HTTPS over plain HTTP. -func (s *Service) LookupPullEndpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) LookupPullEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { return s.lookupEndpoints(repoName) } // LookupPushEndpoints creates an list of endpoints to try to push to, in order of preference. // It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP. // Mirrors are not included. -func (s *Service) LookupPushEndpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) LookupPushEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { allEndpoints, err := s.lookupEndpoints(repoName) if err == nil { for _, endpoint := range allEndpoints { @@ -142,7 +170,7 @@ func (s *Service) LookupPushEndpoints(repoName string) (endpoints []APIEndpoint, return endpoints, err } -func (s *Service) lookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { endpoints, err = s.lookupV2Endpoints(repoName) if err != nil { return nil, err diff --git a/registry/service_v1.go b/registry/service_v1.go index ddb78ee60ab4e..5fdc1ececf311 100644 --- a/registry/service_v1.go +++ b/registry/service_v1.go @@ -4,13 +4,15 @@ import ( "fmt" "strings" + "github.com/docker/distribution/reference" "github.com/docker/docker/pkg/tlsconfig" ) -func (s *Service) lookupV1Endpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - if strings.HasPrefix(repoName, DefaultNamespace+"/") { + nameString := repoName.Name() + if strings.HasPrefix(nameString, DefaultNamespace+"/") { endpoints = append(endpoints, APIEndpoint{ URL: DefaultV1Registry, Version: APIVersion1, @@ -21,11 +23,11 @@ func (s *Service) lookupV1Endpoints(repoName string) (endpoints []APIEndpoint, e return endpoints, nil } - slashIndex := strings.IndexRune(repoName, '/') + slashIndex := strings.IndexRune(nameString, '/') if slashIndex <= 0 { - return nil, fmt.Errorf("invalid repo name: missing '/': %s", repoName) + return nil, fmt.Errorf("invalid repo name: missing '/': %s", nameString) } - hostname := repoName[:slashIndex] + hostname := nameString[:slashIndex] tlsConfig, err = s.TLSConfig(hostname) if err != nil { diff --git a/registry/service_v2.go b/registry/service_v2.go index 70d5fd710e67c..56a3d2eeede31 100644 --- a/registry/service_v2.go +++ b/registry/service_v2.go @@ -4,14 +4,16 @@ import ( "fmt" "strings" + "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/pkg/tlsconfig" ) -func (s *Service) lookupV2Endpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - if strings.HasPrefix(repoName, DefaultNamespace+"/") { + nameString := repoName.Name() + if strings.HasPrefix(nameString, DefaultNamespace+"/") { // v2 mirrors for _, mirror := range s.Config.Mirrors { mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) @@ -39,11 +41,11 @@ func (s *Service) lookupV2Endpoints(repoName string) (endpoints []APIEndpoint, e return endpoints, nil } - slashIndex := strings.IndexRune(repoName, '/') + slashIndex := strings.IndexRune(nameString, '/') if slashIndex <= 0 { - return nil, fmt.Errorf("invalid repo name: missing '/': %s", repoName) + return nil, fmt.Errorf("invalid repo name: missing '/': %s", nameString) } - hostname := repoName[:slashIndex] + hostname := nameString[:slashIndex] tlsConfig, err = s.TLSConfig(hostname) if err != nil { diff --git a/registry/session.go b/registry/session.go index 2a20d321903c7..645e5d44b3ff8 100644 --- a/registry/session.go +++ b/registry/session.go @@ -20,6 +20,7 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/reference" "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" @@ -320,7 +321,9 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io // repository. It queries each of the registries supplied in the registries // argument, and returns data from the first one that answers the query // successfully. -func (r *Session) GetRemoteTag(registries []string, repository string, askedTag string) (string, error) { +func (r *Session) GetRemoteTag(registries []string, repositoryRef reference.Named, askedTag string) (string, error) { + repository := repositoryRef.Name() + if strings.Count(repository, "/") == 0 { // This will be removed once the registry supports auto-resolution on // the "library" namespace @@ -356,7 +359,9 @@ func (r *Session) GetRemoteTag(registries []string, repository string, askedTag // of the registries supplied in the registries argument, and returns data from // the first one that answers the query successfully. It returns a map with // tag names as the keys and image IDs as the values. -func (r *Session) GetRemoteTags(registries []string, repository string) (map[string]string, error) { +func (r *Session) GetRemoteTags(registries []string, repositoryRef reference.Named) (map[string]string, error) { + repository := repositoryRef.Name() + if strings.Count(repository, "/") == 0 { // This will be removed once the registry supports auto-resolution on // the "library" namespace @@ -408,8 +413,8 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { } // GetRepositoryData returns lists of images and endpoints for the repository -func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote) +func (r *Session) GetRepositoryData(remote reference.Named) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote.Name()) logrus.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -443,7 +448,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote.Name(), errBody), res) } var endpoints []string @@ -595,10 +600,10 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry // PushRegistryTag pushes a tag on the registry. // Remote has the format '/ -func (r *Session) PushRegistryTag(remote, revision, tag, registry string) error { +func (r *Session) PushRegistryTag(remote reference.Named, revision, tag, registry string) error { // "jsonify" the string revision = "\"" + revision + "\"" - path := fmt.Sprintf("repositories/%s/tags/%s", remote, tag) + path := fmt.Sprintf("repositories/%s/tags/%s", remote.Name(), tag) req, err := http.NewRequest("PUT", registry+path, strings.NewReader(revision)) if err != nil { @@ -612,13 +617,13 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string) error } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return httputils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote.Name()), res) } return nil } // PushImageJSONIndex uploads an image list to the repository -func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { +func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} if validate { for _, elem := range imgList { @@ -638,7 +643,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote.Name(), suffix) logrus.Debugf("[registry] PUT %s", u) logrus.Debugf("Image list pushed to index:\n%s", imgListJSON) headers := map[string][]string{ @@ -676,7 +681,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote.Name(), errBody), res) } tokens = res.Header["X-Docker-Token"] logrus.Debugf("Auth token: %v", tokens) @@ -694,7 +699,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote.Name(), errBody), res) } } diff --git a/registry/types.go b/registry/types.go index 09b9d57134261..8a201a9172bbc 100644 --- a/registry/types.go +++ b/registry/types.go @@ -1,5 +1,9 @@ package registry +import ( + "github.com/docker/distribution/reference" +) + // SearchResult describes a search result returned from a registry type SearchResult struct { // StarCount indicates the number of stars this repository has @@ -126,13 +130,13 @@ type RepositoryInfo struct { Index *IndexInfo // RemoteName is the remote name of the repository, such as // "library/ubuntu-12.04-base" - RemoteName string + RemoteName reference.Named // LocalName is the local name of the repository, such as // "ubuntu-12.04-base" - LocalName string + LocalName reference.Named // CanonicalName is the canonical name of the repository, such as // "docker.io/library/ubuntu-12.04-base" - CanonicalName string + CanonicalName reference.Named // Official indicates whether the repository is considered official. // If the registry is official, and the normalized name does not // contain a '/' (e.g. "foo"), then it is considered an official repo. diff --git a/utils/utils.go b/utils/utils.go index d2e83f61a7a6d..91246eafce897 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -269,23 +269,6 @@ func ReadDockerIgnore(reader io.ReadCloser) ([]string, error) { return excludes, nil } -// ImageReference combines `repo` and `ref` and returns a string representing -// the combination. If `ref` is a digest (meaning it's of the form -// :, the returned string is @. Otherwise, -// ref is assumed to be a tag, and the returned string is :. -func ImageReference(repo, ref string) string { - if DigestReference(ref) { - return repo + "@" + ref - } - return repo + ":" + ref -} - -// DigestReference returns true if ref is a digest reference; i.e. if it -// is of the form :. -func DigestReference(ref string) bool { - return strings.Contains(ref, ":") -} - // GetErrorMessage returns the human readable message associated with // the passed-in error. In some cases the default Error() func returns // something that is less than useful so based on its types this func diff --git a/utils/utils_test.go b/utils/utils_test.go index 9acb8017a6312..3dc5af88afbd7 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -26,36 +26,6 @@ func TestReplaceAndAppendEnvVars(t *testing.T) { } } -func TestImageReference(t *testing.T) { - tests := []struct { - repo string - ref string - expected string - }{ - {"repo", "tag", "repo:tag"}, - {"repo", "sha256:c100b11b25d0cacd52c14e0e7bf525e1a4c0e6aec8827ae007055545909d1a64", "repo@sha256:c100b11b25d0cacd52c14e0e7bf525e1a4c0e6aec8827ae007055545909d1a64"}, - } - - for i, test := range tests { - actual := ImageReference(test.repo, test.ref) - if test.expected != actual { - t.Errorf("%d: expected %q, got %q", i, test.expected, actual) - } - } -} - -func TestDigestReference(t *testing.T) { - input := "sha256:c100b11b25d0cacd52c14e0e7bf525e1a4c0e6aec8827ae007055545909d1a64" - if !DigestReference(input) { - t.Errorf("Expected DigestReference=true for input %q", input) - } - - input = "latest" - if DigestReference(input) { - t.Errorf("Unexpected DigestReference=true for input %q", input) - } -} - func TestReadDockerIgnore(t *testing.T) { tmpDir, err := ioutil.TempDir("", "dockerignore-test") if err != nil {