Skip to content

Commit

Permalink
Abstract distribution interfaces from image specific types
Browse files Browse the repository at this point in the history
Move configurations into a single file.
Abstract download manager in pull config.
Add supports for schema2 only and schema2 type checking.
Add interface for providing push layers.
Abstract image store to generically handle configurations.

Signed-off-by: Derek McGowan <[email protected]>
  • Loading branch information
dmcgowan committed Dec 19, 2016
1 parent 61ac7c4 commit 3c7676a
Show file tree
Hide file tree
Showing 13 changed files with 457 additions and 212 deletions.
2 changes: 1 addition & 1 deletion cli/command/image/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func runPull(dockerCli *command.DockerCli, opts pullOptions) error {
err = imagePullPrivileged(ctx, dockerCli, authConfig, distributionRef.String(), requestPrivilege, opts.all)
}
if err != nil {
if strings.Contains(err.Error(), "target is a plugin") {
if strings.Contains(err.Error(), "target is plugin") {
return errors.New(err.Error() + " - Use `docker plugin install`")
}
return err
Expand Down
21 changes: 12 additions & 9 deletions daemon/image_pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,15 +89,18 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference.
}()

imagePullConfig := &distribution.ImagePullConfig{
MetaHeaders: metaHeaders,
AuthConfig: authConfig,
ProgressOutput: progress.ChanOutput(progressChan),
RegistryService: daemon.RegistryService,
ImageEventLogger: daemon.LogImageEvent,
MetadataStore: daemon.distributionMetadataStore,
ImageStore: daemon.imageStore,
ReferenceStore: daemon.referenceStore,
DownloadManager: daemon.downloadManager,
Config: distribution.Config{
MetaHeaders: metaHeaders,
AuthConfig: authConfig,
ProgressOutput: progress.ChanOutput(progressChan),
RegistryService: daemon.RegistryService,
ImageEventLogger: daemon.LogImageEvent,
MetadataStore: daemon.distributionMetadataStore,
ImageStore: distribution.NewImageConfigStoreFromStore(daemon.imageStore),
ReferenceStore: daemon.referenceStore,
},
DownloadManager: daemon.downloadManager,
Schema2Types: distribution.ImageTypes,
}

err := distribution.Pull(ctx, ref, imagePullConfig)
Expand Down
26 changes: 15 additions & 11 deletions daemon/image_push.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package daemon
import (
"io"

"github.com/docker/distribution/manifest/schema2"
"github.com/docker/docker/api/types"
"github.com/docker/docker/distribution"
"github.com/docker/docker/pkg/progress"
Expand Down Expand Up @@ -38,17 +39,20 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead
}()

imagePushConfig := &distribution.ImagePushConfig{
MetaHeaders: metaHeaders,
AuthConfig: authConfig,
ProgressOutput: progress.ChanOutput(progressChan),
RegistryService: daemon.RegistryService,
ImageEventLogger: daemon.LogImageEvent,
MetadataStore: daemon.distributionMetadataStore,
LayerStore: daemon.layerStore,
ImageStore: daemon.imageStore,
ReferenceStore: daemon.referenceStore,
TrustKey: daemon.trustKey,
UploadManager: daemon.uploadManager,
Config: distribution.Config{
MetaHeaders: metaHeaders,
AuthConfig: authConfig,
ProgressOutput: progress.ChanOutput(progressChan),
RegistryService: daemon.RegistryService,
ImageEventLogger: daemon.LogImageEvent,
MetadataStore: daemon.distributionMetadataStore,
ImageStore: distribution.NewImageConfigStoreFromStore(daemon.imageStore),
ReferenceStore: daemon.referenceStore,
},
ConfigMediaType: schema2.MediaTypeImageConfig,
LayerStore: distribution.NewLayerProviderFromStore(daemon.layerStore),
TrustKey: daemon.trustKey,
UploadManager: daemon.uploadManager,
}

err = distribution.Push(ctx, ref, imagePushConfig)
Expand Down
233 changes: 233 additions & 0 deletions distribution/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
package distribution

import (
"encoding/json"
"fmt"
"io"
"runtime"

"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/docker/api/types"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/distribution/xfer"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/reference"
"github.com/docker/docker/registry"
"github.com/docker/libtrust"
"golang.org/x/net/context"
)

// Config stores configuration for communicating
// with a registry.
type Config struct {
// MetaHeaders stores HTTP headers with metadata about the image
MetaHeaders map[string][]string
// AuthConfig holds authentication credentials for authenticating with
// the registry.
AuthConfig *types.AuthConfig
// ProgressOutput is the interface for showing the status of the pull
// operation.
ProgressOutput progress.Output
// RegistryService is the registry service to use for TLS configuration
// and endpoint lookup.
RegistryService registry.Service
// ImageEventLogger notifies events for a given image
ImageEventLogger func(id, name, action string)
// MetadataStore is the storage backend for distribution-specific
// metadata.
MetadataStore metadata.Store
// ImageStore manages images.
ImageStore ImageConfigStore
// ReferenceStore manages tags. This value is optional, when excluded
// content will not be tagged.
ReferenceStore reference.Store
// RequireSchema2 ensures that only schema2 manifests are used.
RequireSchema2 bool
}

// ImagePullConfig stores pull configuration.
type ImagePullConfig struct {
Config

// DownloadManager manages concurrent pulls.
DownloadManager RootFSDownloadManager
// Schema2Types is the valid schema2 configuration types allowed
// by the pull operation.
Schema2Types []string
}

// ImagePushConfig stores push configuration.
type ImagePushConfig struct {
Config

// ConfigMediaType is the configuration media type for
// schema2 manifests.
ConfigMediaType string
// LayerStore manages layers.
LayerStore PushLayerProvider
// TrustKey is the private key for legacy signatures. This is typically
// an ephemeral key, since these signatures are no longer verified.
TrustKey libtrust.PrivateKey
// UploadManager dispatches uploads.
UploadManager *xfer.LayerUploadManager
}

// ImageConfigStore handles storing and getting image configurations
// by digest. Allows getting an image configurations rootfs from the
// configuration.
type ImageConfigStore interface {
Put([]byte) (digest.Digest, error)
Get(digest.Digest) ([]byte, error)
RootFSFromConfig([]byte) (*image.RootFS, error)
}

// PushLayerProvider provides layers to be pushed by ChainID.
type PushLayerProvider interface {
Get(layer.ChainID) (PushLayer, error)
}

// PushLayer is a pushable layer with metadata about the layer
// and access to the content of the layer.
type PushLayer interface {
ChainID() layer.ChainID
DiffID() layer.DiffID
Parent() PushLayer
Open() (io.ReadCloser, error)
Size() (int64, error)
MediaType() string
Release()
}

// RootFSDownloadManager handles downloading of the rootfs
type RootFSDownloadManager interface {
// Download downloads the layers into the given initial rootfs and
// returns the final rootfs.
// Given progress output to track download progress
// Returns function to release download resources
Download(ctx context.Context, initialRootFS image.RootFS, layers []xfer.DownloadDescriptor, progressOutput progress.Output) (image.RootFS, func(), error)
}

type imageConfigStore struct {
image.Store
}

// NewImageConfigStoreFromStore returns an ImageConfigStore backed
// by an image.Store for container images.
func NewImageConfigStoreFromStore(is image.Store) ImageConfigStore {
return &imageConfigStore{
Store: is,
}
}

func (s *imageConfigStore) Put(c []byte) (digest.Digest, error) {
id, err := s.Store.Create(c)
return digest.Digest(id), err
}

func (s *imageConfigStore) Get(d digest.Digest) ([]byte, error) {
img, err := s.Store.Get(image.IDFromDigest(d))
if err != nil {
return nil, err
}
return img.RawJSON(), nil
}

func (s *imageConfigStore) RootFSFromConfig(c []byte) (*image.RootFS, error) {
var unmarshalledConfig image.Image
if err := json.Unmarshal(c, &unmarshalledConfig); err != nil {
return nil, err
}

// fail immediately on windows
if runtime.GOOS == "windows" && unmarshalledConfig.OS == "linux" {
return nil, fmt.Errorf("image operating system %q cannot be used on this platform", unmarshalledConfig.OS)
}

return unmarshalledConfig.RootFS, nil
}

type storeLayerProvider struct {
ls layer.Store
}

// NewLayerProviderFromStore returns a layer provider backed by
// an instance of LayerStore. Only getting layers as gzipped
// tars is supported.
func NewLayerProviderFromStore(ls layer.Store) PushLayerProvider {
return &storeLayerProvider{
ls: ls,
}
}

func (p *storeLayerProvider) Get(lid layer.ChainID) (PushLayer, error) {
if lid == "" {
return &storeLayer{
Layer: layer.EmptyLayer,
}, nil
}
l, err := p.ls.Get(lid)
if err != nil {
return nil, err
}

sl := storeLayer{
Layer: l,
ls: p.ls,
}
if d, ok := l.(distribution.Describable); ok {
return &describableStoreLayer{
storeLayer: sl,
describable: d,
}, nil
}

return &sl, nil
}

type storeLayer struct {
layer.Layer
ls layer.Store
}

func (l *storeLayer) Parent() PushLayer {
p := l.Layer.Parent()
if p == nil {
return nil
}
return &storeLayer{
Layer: p,
ls: l.ls,
}
}

func (l *storeLayer) Open() (io.ReadCloser, error) {
return l.Layer.TarStream()
}

func (l *storeLayer) Size() (int64, error) {
return l.Layer.DiffSize()
}

func (l *storeLayer) MediaType() string {
// layer store always returns uncompressed tars
return schema2.MediaTypeUncompressedLayer
}

func (l *storeLayer) Release() {
if l.ls != nil {
layer.ReleaseAndLog(l.ls, l.Layer)
}
}

type describableStoreLayer struct {
storeLayer
describable distribution.Describable
}

func (l *describableStoreLayer) Descriptor() distribution.Descriptor {
return l.describable.Descriptor()
}
33 changes: 4 additions & 29 deletions distribution/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,42 +6,13 @@ import (
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/api"
"github.com/docker/docker/api/types"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/distribution/xfer"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/reference"
"github.com/docker/docker/registry"
"golang.org/x/net/context"
)

// ImagePullConfig stores pull configuration.
type ImagePullConfig struct {
// MetaHeaders stores HTTP headers with metadata about the image
MetaHeaders map[string][]string
// AuthConfig holds authentication credentials for authenticating with
// the registry.
AuthConfig *types.AuthConfig
// ProgressOutput is the interface for showing the status of the pull
// operation.
ProgressOutput progress.Output
// RegistryService is the registry service to use for TLS configuration
// and endpoint lookup.
RegistryService registry.Service
// ImageEventLogger notifies events for a given image
ImageEventLogger func(id, name, action string)
// MetadataStore is the storage backend for distribution-specific
// metadata.
MetadataStore metadata.Store
// ImageStore manages images.
ImageStore image.Store
// ReferenceStore manages tags.
ReferenceStore reference.Store
// DownloadManager manages concurrent pulls.
DownloadManager *xfer.LayerDownloadManager
}

// Puller is an interface that abstracts pulling for different API versions.
type Puller interface {
// Pull tries to pull the image referenced by `tag`
Expand Down Expand Up @@ -117,6 +88,10 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo
confirmedTLSRegistries = make(map[string]struct{})
)
for _, endpoint := range endpoints {
if imagePullConfig.RequireSchema2 && endpoint.Version == registry.APIVersion1 {
continue
}

if confirmedV2 && endpoint.Version == registry.APIVersion1 {
logrus.Debugf("Skipping v1 endpoint %s because v2 registry was detected", endpoint.URL)
continue
Expand Down
8 changes: 5 additions & 3 deletions distribution/pull_v1.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,13 +243,15 @@ func (p *v1Puller) pullImage(ctx context.Context, v1ID, endpoint string, localNa
return err
}

imageID, err := p.config.ImageStore.Create(config)
imageID, err := p.config.ImageStore.Put(config)
if err != nil {
return err
}

if err := p.config.ReferenceStore.AddTag(localNameRef, imageID.Digest(), true); err != nil {
return err
if p.config.ReferenceStore != nil {
if err := p.config.ReferenceStore.AddTag(localNameRef, imageID, true); err != nil {
return err
}
}

return nil
Expand Down
Loading

0 comments on commit 3c7676a

Please sign in to comment.