diff --git a/pkg/content/consts.go b/pkg/content/consts.go index 874aa1691..bda9e5f81 100644 --- a/pkg/content/consts.go +++ b/pkg/content/consts.go @@ -20,3 +20,9 @@ const ( // AnnotationUnpack is the annotation key for indication of unpacking AnnotationUnpack = "io.deis.oras.content.unpack" ) + +const ( + // OCIImageIndexFile is the file name of the index from the OCI Image Layout Specification + // Reference: https://github.com/opencontainers/image-spec/blob/master/image-layout.md#indexjson-file + OCIImageIndexFile = "index.json" +) diff --git a/pkg/content/errors.go b/pkg/content/errors.go index f772b1ffe..e4a6cbf45 100644 --- a/pkg/content/errors.go +++ b/pkg/content/errors.go @@ -4,9 +4,10 @@ import "errors" // Common errors var ( - ErrNotFound = errors.New("not_found") - ErrNoName = errors.New("no_name") - ErrUnsupportedSize = errors.New("unsupported_size") + ErrNotFound = errors.New("not_found") + ErrNoName = errors.New("no_name") + ErrUnsupportedSize = errors.New("unsupported_size") + ErrUnsupportedVersion = errors.New("unsupported_version") ) // FileStore errors diff --git a/pkg/content/file.go b/pkg/content/file.go index 3e5b7fde2..85bb08811 100644 --- a/pkg/content/file.go +++ b/pkg/content/file.go @@ -20,8 +20,7 @@ import ( // ensure interface var ( - _ content.Provider = &FileStore{} - _ content.Ingester = &FileStore{} + _ ProvideIngester = &FileStore{} ) // FileStore provides content from the file system diff --git a/pkg/content/interface.go b/pkg/content/interface.go new file mode 100644 index 000000000..85caa3fd8 --- /dev/null +++ b/pkg/content/interface.go @@ -0,0 +1,9 @@ +package content + +import "github.com/containerd/containerd/content" + +// ProvideIngester is the interface that groups the basic Read and Write methods. +type ProvideIngester interface { + content.Provider + content.Ingester +} diff --git a/pkg/content/oci.go b/pkg/content/oci.go new file mode 100644 index 000000000..57c56ef17 --- /dev/null +++ b/pkg/content/oci.go @@ -0,0 +1,169 @@ +package content + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/content/local" + specs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// OCIStore provides content from the file system with the OCI-Image layout. +// Reference: https://github.com/opencontainers/image-spec/blob/master/image-layout.md +type OCIStore struct { + content.Store + + root string + index *ocispec.Index + nameMap map[string]ocispec.Descriptor +} + +// NewOCIStore creates a new OCI store +func NewOCIStore(rootPath string) (*OCIStore, error) { + fileStore, err := local.NewStore(rootPath) + if err != nil { + return nil, err + } + + store := &OCIStore{ + Store: fileStore, + root: rootPath, + } + if err := store.validateOCILayoutFile(); err != nil { + return nil, err + } + if err := store.LoadIndex(); err != nil { + return nil, err + } + + return store, nil +} + +// LoadIndex reads the index.json from the file system +func (s *OCIStore) LoadIndex() error { + path := filepath.Join(s.root, OCIImageIndexFile) + indexFile, err := os.Open(path) + if err != nil { + if !os.IsNotExist(err) { + return err + } + s.index = &ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value + }, + } + s.nameMap = make(map[string]ocispec.Descriptor) + + return nil + } + defer indexFile.Close() + + if err := json.NewDecoder(indexFile).Decode(&s.index); err != nil { + return err + } + + s.nameMap = make(map[string]ocispec.Descriptor) + for _, desc := range s.index.Manifests { + if name := desc.Annotations[ocispec.AnnotationRefName]; name != "" { + s.nameMap[name] = desc + } + } + + return nil +} + +// SaveIndex writes the index.json to the file system +func (s *OCIStore) SaveIndex() error { + indexJSON, err := json.Marshal(s.index) + if err != nil { + return err + } + + path := filepath.Join(s.root, OCIImageIndexFile) + return ioutil.WriteFile(path, indexJSON, 0644) +} + +// AddReference adds or updates an reference to index. +func (s *OCIStore) AddReference(name string, desc ocispec.Descriptor) { + if desc.Annotations == nil { + desc.Annotations = map[string]string{ + ocispec.AnnotationRefName: name, + } + } else { + desc.Annotations[ocispec.AnnotationRefName] = name + } + + if _, ok := s.nameMap[name]; ok { + for i, ref := range s.index.Manifests { + if name == ref.Annotations[ocispec.AnnotationRefName] { + s.index.Manifests[i] = desc + return + } + } + + // Process should not reach here. + // Fallthrough to `Add` scenario and recover. + } + + s.index.Manifests = append(s.index.Manifests, desc) + s.nameMap[name] = desc + return +} + +// DeleteReference deletes an reference from index. +func (s *OCIStore) DeleteReference(name string) { + if _, ok := s.nameMap[name]; !ok { + return + } + + delete(s.nameMap, name) + for i, desc := range s.index.Manifests { + if name == desc.Annotations[ocispec.AnnotationRefName] { + s.index.Manifests[i] = s.index.Manifests[len(s.index.Manifests)-1] + s.index.Manifests = s.index.Manifests[:len(s.index.Manifests)-1] + return + } + } +} + +// ListReferences lists all references in index. +func (s *OCIStore) ListReferences() map[string]ocispec.Descriptor { + return s.nameMap +} + +// validateOCILayoutFile ensures the `oci-layout` file +func (s *OCIStore) validateOCILayoutFile() error { + layoutFilePath := filepath.Join(s.root, ocispec.ImageLayoutFile) + layoutFile, err := os.Open(layoutFilePath) + if err != nil { + if !os.IsNotExist(err) { + return err + } + + layout := ocispec.ImageLayout{ + Version: ocispec.ImageLayoutVersion, + } + layoutJSON, err := json.Marshal(layout) + if err != nil { + return err + } + + return ioutil.WriteFile(layoutFilePath, layoutJSON, 0644) + } + defer layoutFile.Close() + + var layout *ocispec.ImageLayout + err = json.NewDecoder(layoutFile).Decode(&layout) + if err != nil { + return err + } + if layout.Version != ocispec.ImageLayoutVersion { + return ErrUnsupportedVersion + } + + return nil +} diff --git a/pkg/oras/pull.go b/pkg/oras/pull.go index 2437000f8..ebea35e1f 100644 --- a/pkg/oras/pull.go +++ b/pkg/oras/pull.go @@ -48,7 +48,7 @@ func fetchContent(ctx context.Context, fetcher remotes.Fetcher, desc ocispec.Des lock := &sync.Mutex{} picker := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { if isAllowedMediaType(desc.MediaType, opts.allowedMediaTypes...) { - if name, ok := orascontent.ResolveName(desc); ok && len(name) > 0 { + if opts.filterName(desc) { lock.Lock() defer lock.Unlock() descriptors = append(descriptors, desc) @@ -57,9 +57,13 @@ func fetchContent(ctx context.Context, fetcher remotes.Fetcher, desc ocispec.Des } return nil, nil }) - store := newHybridStoreFromIngester(ingester) + + store := opts.contentProvideIngester + if store == nil { + store = newHybridStoreFromIngester(ingester) + } handlers := []images.Handler{ - filterHandler(opts.allowedMediaTypes...), + filterHandler(opts, opts.allowedMediaTypes...), } handlers = append(handlers, opts.baseHandlers...) handlers = append(handlers, @@ -76,13 +80,13 @@ func fetchContent(ctx context.Context, fetcher remotes.Fetcher, desc ocispec.Des return descriptors, nil } -func filterHandler(allowedMediaTypes ...string) images.HandlerFunc { +func filterHandler(opts *pullOpts, allowedMediaTypes ...string) images.HandlerFunc { return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch { case isAllowedMediaType(desc.MediaType, ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex): return nil, nil case isAllowedMediaType(desc.MediaType, allowedMediaTypes...): - if name, ok := orascontent.ResolveName(desc); ok && len(name) > 0 { + if opts.filterName(desc) { return nil, nil } log.G(ctx).Warnf("blob no name: %v", desc.Digest) @@ -123,3 +127,8 @@ func dispatchBFS(ctx context.Context, handler images.Handler, descs ...ocispec.D } return nil } + +func filterName(desc ocispec.Descriptor) bool { + name, ok := orascontent.ResolveName(desc) + return ok && len(name) > 0 +} diff --git a/pkg/oras/pull_opts.go b/pkg/oras/pull_opts.go index 40a35ea7e..eb50fd470 100644 --- a/pkg/oras/pull_opts.go +++ b/pkg/oras/pull_opts.go @@ -3,15 +3,19 @@ package oras import ( "context" + orascontent "github.com/deislabs/oras/pkg/content" + "github.com/containerd/containerd/images" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) type pullOpts struct { - allowedMediaTypes []string - dispatch func(context.Context, images.Handler, ...ocispec.Descriptor) error - baseHandlers []images.Handler - callbackHandlers []images.Handler + allowedMediaTypes []string + dispatch func(context.Context, images.Handler, ...ocispec.Descriptor) error + baseHandlers []images.Handler + callbackHandlers []images.Handler + contentProvideIngester orascontent.ProvideIngester + filterName func(ocispec.Descriptor) bool } // PullOpt allows callers to set options on the oras pull @@ -19,7 +23,8 @@ type PullOpt func(o *pullOpts) error func pullOptsDefaults() *pullOpts { return &pullOpts{ - dispatch: images.Dispatch, + dispatch: images.Dispatch, + filterName: filterName, } } @@ -62,3 +67,22 @@ func WithPullCallbackHandler(handlers ...images.Handler) PullOpt { return nil } } + +// WithContentProvideIngester opt to the provided Provider and Ingester +// for file system I/O, including caches. +func WithContentProvideIngester(store orascontent.ProvideIngester) PullOpt { + return func(o *pullOpts) error { + o.contentProvideIngester = store + return nil + } +} + +// WithPullEmptyNameAllowed allows pulling blobs with empty name. +func WithPullEmptyNameAllowed() PullOpt { + return func(o *pullOpts) error { + o.filterName = func(ocispec.Descriptor) bool { + return true + } + return nil + } +}