New OCIStore (oras-project#108)

* Add basic OCIStore

* load index at startup

* CRUD for index

* user provided provider and ingester

* simplify code

* allow pulling blobs with no names

pkg/content/consts.go

@@ -20,3 +20,9 @@ const (
 	// AnnotationUnpack is the annotation key for indication of unpacking
 	AnnotationUnpack = "io.deis.oras.content.unpack"
 )
+
+const (
+	// OCIImageIndexFile is the file name of the index from the OCI Image Layout Specification
+	// Reference: https://github.com/opencontainers/image-spec/blob/master/image-layout.md#indexjson-file
+	OCIImageIndexFile = "index.json"
+)

pkg/content/errors.go

@@ -4,9 +4,10 @@ import "errors"
 
 // Common errors
 var (
-	ErrNotFound        = errors.New("not_found")
-	ErrNoName          = errors.New("no_name")
-	ErrUnsupportedSize = errors.New("unsupported_size")
+	ErrNotFound           = errors.New("not_found")
+	ErrNoName             = errors.New("no_name")
+	ErrUnsupportedSize    = errors.New("unsupported_size")
+	ErrUnsupportedVersion = errors.New("unsupported_version")
 )
 
 // FileStore errors

pkg/content/file.go

@@ -20,8 +20,7 @@ import (
 
 // ensure interface
 var (
-	_ content.Provider = &FileStore{}
-	_ content.Ingester = &FileStore{}
+	_ ProvideIngester = &FileStore{}
 )
 
 // FileStore provides content from the file system

pkg/content/interface.go

@@ -0,0 +1,9 @@
+package content
+
+import "github.com/containerd/containerd/content"
+
+// ProvideIngester is the interface that groups the basic Read and Write methods.
+type ProvideIngester interface {
+	content.Provider
+	content.Ingester
+}

pkg/content/oci.go

@@ -0,0 +1,169 @@
+package content
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/content/local"
+	specs "github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// OCIStore provides content from the file system with the OCI-Image layout.
+// Reference: https://github.com/opencontainers/image-spec/blob/master/image-layout.md
+type OCIStore struct {
+	content.Store
+
+	root    string
+	index   *ocispec.Index
+	nameMap map[string]ocispec.Descriptor
+}
+
+// NewOCIStore creates a new OCI store
+func NewOCIStore(rootPath string) (*OCIStore, error) {
+	fileStore, err := local.NewStore(rootPath)
+	if err != nil {
+		return nil, err
+	}
+
+	store := &OCIStore{
+		Store: fileStore,
+		root:  rootPath,
+	}
+	if err := store.validateOCILayoutFile(); err != nil {
+		return nil, err
+	}
+	if err := store.LoadIndex(); err != nil {
+		return nil, err
+	}
+
+	return store, nil
+}
+
+// LoadIndex reads the index.json from the file system
+func (s *OCIStore) LoadIndex() error {
+	path := filepath.Join(s.root, OCIImageIndexFile)
+	indexFile, err := os.Open(path)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+		s.index = &ocispec.Index{
+			Versioned: specs.Versioned{
+				SchemaVersion: 2, // historical value
+			},
+		}
+		s.nameMap = make(map[string]ocispec.Descriptor)
+
+		return nil
+	}
+	defer indexFile.Close()
+
+	if err := json.NewDecoder(indexFile).Decode(&s.index); err != nil {
+		return err
+	}
+
+	s.nameMap = make(map[string]ocispec.Descriptor)
+	for _, desc := range s.index.Manifests {
+		if name := desc.Annotations[ocispec.AnnotationRefName]; name != "" {
+			s.nameMap[name] = desc
+		}
+	}
+
+	return nil
+}
+
+// SaveIndex writes the index.json to the file system
+func (s *OCIStore) SaveIndex() error {
+	indexJSON, err := json.Marshal(s.index)
+	if err != nil {
+		return err
+	}
+
+	path := filepath.Join(s.root, OCIImageIndexFile)
+	return ioutil.WriteFile(path, indexJSON, 0644)
+}
+
+// AddReference adds or updates an reference to index.
+func (s *OCIStore) AddReference(name string, desc ocispec.Descriptor) {
+	if desc.Annotations == nil {
+		desc.Annotations = map[string]string{
+			ocispec.AnnotationRefName: name,
+		}
+	} else {
+		desc.Annotations[ocispec.AnnotationRefName] = name
+	}
+
+	if _, ok := s.nameMap[name]; ok {
+		for i, ref := range s.index.Manifests {
+			if name == ref.Annotations[ocispec.AnnotationRefName] {
+				s.index.Manifests[i] = desc
+				return
+			}
+		}
+
+		// Process should not reach here.
+		// Fallthrough to `Add` scenario and recover.
+	}
+
+	s.index.Manifests = append(s.index.Manifests, desc)
+	s.nameMap[name] = desc
+	return
+}
+
+// DeleteReference deletes an reference from index.
+func (s *OCIStore) DeleteReference(name string) {
+	if _, ok := s.nameMap[name]; !ok {
+		return
+	}
+
+	delete(s.nameMap, name)
+	for i, desc := range s.index.Manifests {
+		if name == desc.Annotations[ocispec.AnnotationRefName] {
+			s.index.Manifests[i] = s.index.Manifests[len(s.index.Manifests)-1]
+			s.index.Manifests = s.index.Manifests[:len(s.index.Manifests)-1]
+			return
+		}
+	}
+}
+
+// ListReferences lists all references in index.
+func (s *OCIStore) ListReferences() map[string]ocispec.Descriptor {
+	return s.nameMap
+}
+
+// validateOCILayoutFile ensures the `oci-layout` file
+func (s *OCIStore) validateOCILayoutFile() error {
+	layoutFilePath := filepath.Join(s.root, ocispec.ImageLayoutFile)
+	layoutFile, err := os.Open(layoutFilePath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+
+		layout := ocispec.ImageLayout{
+			Version: ocispec.ImageLayoutVersion,
+		}
+		layoutJSON, err := json.Marshal(layout)
+		if err != nil {
+			return err
+		}
+
+		return ioutil.WriteFile(layoutFilePath, layoutJSON, 0644)
+	}
+	defer layoutFile.Close()
+
+	var layout *ocispec.ImageLayout
+	err = json.NewDecoder(layoutFile).Decode(&layout)
+	if err != nil {
+		return err
+	}
+	if layout.Version != ocispec.ImageLayoutVersion {
+		return ErrUnsupportedVersion
+	}
+
+	return nil
+}

pkg/oras/pull.go

@@ -48,7 +48,7 @@ func fetchContent(ctx context.Context, fetcher remotes.Fetcher, desc ocispec.Des
 	lock := &sync.Mutex{}
 	picker := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 		if isAllowedMediaType(desc.MediaType, opts.allowedMediaTypes...) {
-			if name, ok := orascontent.ResolveName(desc); ok && len(name) > 0 {
+			if opts.filterName(desc) {
 				lock.Lock()
 				defer lock.Unlock()
 				descriptors = append(descriptors, desc)
@@ -57,9 +57,13 @@ func fetchContent(ctx context.Context, fetcher remotes.Fetcher, desc ocispec.Des
 		}
 		return nil, nil
 	})
-	store := newHybridStoreFromIngester(ingester)
+
+	store := opts.contentProvideIngester
+	if store == nil {
+		store = newHybridStoreFromIngester(ingester)
+	}
 	handlers := []images.Handler{
-		filterHandler(opts.allowedMediaTypes...),
+		filterHandler(opts, opts.allowedMediaTypes...),
 	}
 	handlers = append(handlers, opts.baseHandlers...)
 	handlers = append(handlers,
@@ -76,13 +80,13 @@ func fetchContent(ctx context.Context, fetcher remotes.Fetcher, desc ocispec.Des
 	return descriptors, nil
 }
 
-func filterHandler(allowedMediaTypes ...string) images.HandlerFunc {
+func filterHandler(opts *pullOpts, allowedMediaTypes ...string) images.HandlerFunc {
 	return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 		switch {
 		case isAllowedMediaType(desc.MediaType, ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex):
 			return nil, nil
 		case isAllowedMediaType(desc.MediaType, allowedMediaTypes...):
-			if name, ok := orascontent.ResolveName(desc); ok && len(name) > 0 {
+			if opts.filterName(desc) {
 				return nil, nil
 			}
 			log.G(ctx).Warnf("blob no name: %v", desc.Digest)
@@ -123,3 +127,8 @@ func dispatchBFS(ctx context.Context, handler images.Handler, descs ...ocispec.D
 	}
 	return nil
 }
+
+func filterName(desc ocispec.Descriptor) bool {
+	name, ok := orascontent.ResolveName(desc)
+	return ok && len(name) > 0
+}

pkg/oras/pull_opts.go

@@ -3,23 +3,28 @@ package oras
 import (
 	"context"
 
+	orascontent "github.com/deislabs/oras/pkg/content"
+
 	"github.com/containerd/containerd/images"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 type pullOpts struct {
-	allowedMediaTypes []string
-	dispatch          func(context.Context, images.Handler, ...ocispec.Descriptor) error
-	baseHandlers      []images.Handler
-	callbackHandlers  []images.Handler
+	allowedMediaTypes      []string
+	dispatch               func(context.Context, images.Handler, ...ocispec.Descriptor) error
+	baseHandlers           []images.Handler
+	callbackHandlers       []images.Handler
+	contentProvideIngester orascontent.ProvideIngester
+	filterName             func(ocispec.Descriptor) bool
 }
 
 // PullOpt allows callers to set options on the oras pull
 type PullOpt func(o *pullOpts) error
 
 func pullOptsDefaults() *pullOpts {
 	return &pullOpts{
-		dispatch: images.Dispatch,
+		dispatch:   images.Dispatch,
+		filterName: filterName,
 	}
 }
 
@@ -62,3 +67,22 @@ func WithPullCallbackHandler(handlers ...images.Handler) PullOpt {
 		return nil
 	}
 }
+
+// WithContentProvideIngester opt to the provided Provider and Ingester
+// for file system I/O, including caches.
+func WithContentProvideIngester(store orascontent.ProvideIngester) PullOpt {
+	return func(o *pullOpts) error {
+		o.contentProvideIngester = store
+		return nil
+	}
+}
+
+// WithPullEmptyNameAllowed allows pulling blobs with empty name.
+func WithPullEmptyNameAllowed() PullOpt {
+	return func(o *pullOpts) error {
+		o.filterName = func(ocispec.Descriptor) bool {
+			return true
+		}
+		return nil
+	}
+}