diff --git a/mixer/cmd/mixs/cmd/validator.go b/mixer/cmd/mixs/cmd/validator.go index ad54ad7d1ff8..e48de5a06e2e 100644 --- a/mixer/cmd/mixs/cmd/validator.go +++ b/mixer/cmd/mixs/cmd/validator.go @@ -34,7 +34,8 @@ import ( func validatorCmd(info map[string]template.Info, adapters []adapter.InfoFn, printf, fatalf shared.FormatFn) *cobra.Command { vc := crd.ControllerOptions{} var kubeconfig string - kinds := runtime.KindMap(config.InventoryMap(adapters), info) + tmplRepo := template.NewRepository(info) + kinds := runtime.KindMap(config.AdapterInfoMap(adapters, tmplRepo.SupportsTemplate), info) vc.ResourceNames = make([]string, 0, len(kinds)) for name := range kinds { vc.ResourceNames = append(vc.ResourceNames, pluralize(name)) diff --git a/mixer/pkg/config/BUILD b/mixer/pkg/config/BUILD index 29c9f0293b3c..21b41296adc4 100644 --- a/mixer/pkg/config/BUILD +++ b/mixer/pkg/config/BUILD @@ -11,40 +11,21 @@ go_library( visibility = ["//visibility:public"], deps = [ "//mixer/pkg/adapter:go_default_library", - "//mixer/pkg/attribute:go_default_library", "//mixer/pkg/config/crd:go_default_library", - "//mixer/pkg/config/descriptor:go_default_library", - "//mixer/pkg/config/proto:go_default_library", "//mixer/pkg/config/store:go_default_library", - "//mixer/pkg/expr:go_default_library", - "//mixer/pkg/pool:go_default_library", - "//mixer/pkg/status:go_default_library", - "//mixer/pkg/template:go_default_library", "//pkg/log:go_default_library", - "@com_github_emicklei_go_restful//:go_default_library", - "@com_github_ghodss_yaml//:go_default_library", - "@com_github_gogo_protobuf//jsonpb:go_default_library", - "@com_github_gogo_protobuf//proto:go_default_library", - "@com_github_googleapis_googleapis//:google/rpc", - "@com_github_hashicorp_go_multierror//:go_default_library", - "@io_istio_api//mixer/v1/config/descriptor:descriptor", ], ) go_test( name = "go_default_test", size = "small", - srcs = [ - "adapterInfoRegistry_test.go", - ], + srcs = ["adapterInfoRegistry_test.go"], library = ":go_default_library", deps = [ - "//mixer/pkg/il/testing:go_default_library", + "//mixer/pkg/template:go_default_library", "//mixer/template/sample:go_default_library", "//mixer/template/sample/report:go_default_library", "@com_github_gogo_protobuf//types:go_default_library", - "@com_github_golang_protobuf//ptypes/empty:go_default_library", - "@com_github_golang_protobuf//ptypes/wrappers:go_default_library", - "@org_uber_go_zap//zapcore:go_default_library", ], ) diff --git a/mixer/pkg/config/adapterInfoRegistry.go b/mixer/pkg/config/adapterInfoRegistry.go index 021b5f6bec14..6071db8d9c15 100644 --- a/mixer/pkg/config/adapterInfoRegistry.go +++ b/mixer/pkg/config/adapterInfoRegistry.go @@ -95,13 +95,3 @@ func doesBuilderSupportsTemplates(info adapter.Info, hndlrBldrValidator handlerB } return true, "" } - -// InventoryMap converts adapter inventory to a builder map. -func InventoryMap(inv []adapter.InfoFn) map[string]*adapter.Info { - m := make(map[string]*adapter.Info, len(inv)) - for _, ai := range inv { - bi := ai() - m[bi.Name] = &bi - } - return m -} diff --git a/mixer/pkg/config/store/BUILD b/mixer/pkg/config/store/BUILD index 0d00a8feecf6..f466d26f0c9f 100644 --- a/mixer/pkg/config/store/BUILD +++ b/mixer/pkg/config/store/BUILD @@ -6,11 +6,9 @@ go_library( name = "go_default_library", srcs = [ "convert.go", - "fsstore.go", "fsstore2.go", "memstore.go", "queue.go", - "store.go", "store2.go", "validator.go", ], @@ -31,7 +29,6 @@ go_test( "fsstore2_test.go", "queue_test.go", "store2_test.go", - "store_test.go", "validator_test.go", ], library = ":go_default_library", diff --git a/mixer/pkg/config/store/fsstore.go b/mixer/pkg/config/store/fsstore.go deleted file mode 100644 index f366eec8a365..000000000000 --- a/mixer/pkg/config/store/fsstore.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2017 Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package store - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" - - "istio.io/istio/pkg/log" -) - -// fsStore implements file system KeyValueStore and change store. -type fsStore struct { - // root directory - root string - // tmpdir is used as a scratch pad area - tmpdir string - // file suffix - suffix string - - // testing and fault injection - tempFile writeCloserFunc - readfile readFileFunc - mkdirAll mkdirAllFunc - remove removeFunc -} - -// for testing -type writeCloser interface { - io.WriteCloser - Name() string -} -type writeCloserFunc func() (f writeCloser, err error) -type readFileFunc func(filename string) ([]byte, error) -type mkdirAllFunc func(path string, perm os.FileMode) error -type removeFunc func(name string) error - -func newFSStore(root string) (KeyValueStore, error) { - finfo, err := os.Stat(root) - if err != nil { - return nil, err - } - if !finfo.IsDir() { - return nil, fmt.Errorf("%s is not a directory", root) - } - s := &fsStore{ - root: root, - tmpdir: root + "/TMP", - suffix: ".yml", - readfile: ioutil.ReadFile, - mkdirAll: os.MkdirAll, - remove: os.Remove, - } - - s.tempFile = func() (f writeCloser, err error) { - if err = s.mkdirAll(s.tmpdir, os.ModeDir|os.ModePerm); err != nil { - return nil, err - } - f, err = ioutil.TempFile(s.tmpdir, "fsStore") - return f, err - } - return s, nil -} - -func (f *fsStore) String() string { - return fmt.Sprintf("fsStore: %s", f.root) -} - -func (f *fsStore) Close() { - // Do nothing. -} - -func (f *fsStore) getPath(key string) string { - return path.Join(f.root, key) -} - -// Get value at a key, false if not found. -func (f *fsStore) Get(key string) (value string, index int, found bool) { - p := f.getPath(key) + f.suffix - var b []byte - var err error - - if b, err = f.readfile(p); err != nil { - if !os.IsNotExist(err) { - log.Warnf("Could not access '%s': %v", p, err) - } - return "", IndexNotSupported, false - } - return string(b), IndexNotSupported, true -} - -// Set a value -func (f *fsStore) Set(key string, value string) (index int, err error) { - p := f.getPath(key) + f.suffix - if err = f.mkdirAll(filepath.Dir(p), os.ModeDir|os.ModePerm); err != nil { - return IndexNotSupported, err - } - - var tf writeCloser - if tf, err = f.tempFile(); err != nil { - return IndexNotSupported, err - } - - _, err = tf.Write([]byte(value)) - // always close the file and return the 1st failure. - errClose := tf.Close() - if err != nil { - return IndexNotSupported, err - } - if errClose != nil { - return IndexNotSupported, errClose - } - // file has been written and closed. - // atomically rename - return IndexNotSupported, os.Rename(tf.Name(), p) -} - -// List keys with the prefix -func (f *fsStore) List(key string, recurse bool) (keys []string, index int, err error) { - keys = make([]string, 0, 10) - cc := &keys - - err = filepath.Walk(f.getPath(key), func(path string, info os.FileInfo, err error) error { - if strings.HasSuffix(path, f.suffix) { - *cc = append(*cc, path[len(f.root):len(path)-len(f.suffix)]) - } - return nil - }) - return keys, IndexNotSupported, err -} - -// Delete removes a key from the fs store. -func (f *fsStore) Delete(key string) (err error) { - p := f.getPath(key) + f.suffix - if err = f.remove(p); err == nil || os.IsNotExist(err) { - return nil - } - return err -} diff --git a/mixer/pkg/config/store/store.go b/mixer/pkg/config/store/store.go deleted file mode 100644 index 5a6b66c43fe2..000000000000 --- a/mixer/pkg/config/store/store.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2017 Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package store provides the interface to the backend storage -// for the config and the default fsstore implementation. -package store - -import ( - "fmt" - "net/url" -) - -// IndexNotSupported will be used as the returned index value when -// the KeyValueStore implementation does not support index. -const IndexNotSupported = -1 - -// Builder is the type of function to build a KeyValueStore. -type Builder func(u *url.URL) (KeyValueStore, error) - -// RegisterFunc is the type to register a builder for URL scheme. -type RegisterFunc func(map[string]Builder) - -// ChangeType denotes the type of a change -type ChangeType int - -const ( - // Update - change was an update or a create to a key. - Update ChangeType = iota - // Delete - key was removed. - Delete -) - -// Change - A record of mutation to the underlying KeyValueStore. -type Change struct { - // Key that was affected - Key string `json:"key"` - // Type how did the key change - Type ChangeType `json:"change_type"` - // change log index number of the change - Index int `json:"index"` -} - -// KeyValueStore defines the key value store back end interface used by mixer -// and Mixer config API server. -// -// It should support back ends like redis, etcd and NFS -// All commands should return a change log index number which can be used -// to Read changes. If a KeyValueStore does not support it, -// it should return -1 -type KeyValueStore interface { - // Get value at a key, false if not found. - Get(key string) (value string, index int, found bool) - - // Set a value. - Set(key string, value string) (index int, err error) - - // List keys with the prefix. - List(key string, recurse bool) (keys []string, index int, err error) - - // Delete a key. - Delete(key string) error - - // Close the storage. - Close() - - fmt.Stringer -} - -// ChangeLogReader read change log from the KV Store -type ChangeLogReader interface { - // Read reads change events >= index - Read(index int) ([]Change, error) -} - -// ChangeNotifier implements change notification machinery for the KeyValueStore. -type ChangeNotifier interface { - // Register StoreListener - // KeyValueStore should call this method when there is a change - // The client should issue ReadChangeLog to see what has changed if the call is available. - // else it should re-read the store, perform diff and apply changes. - RegisterListener(s Listener) -} - -// Listener listens for calls from the store that some keys have changed. -type Listener interface { - // NotifyStoreChanged notify listener that a new change is available. - NotifyStoreChanged(index int) -} - -// Registry keeps the relationship between the URL scheme and the storage -// implementation. -type Registry struct { - builders map[string]Builder -} - -// NewRegistry creates a new Registry instance for the inventory. -func NewRegistry(inventory ...RegisterFunc) *Registry { - b := map[string]Builder{} - for _, rf := range inventory { - rf(b) - } - return &Registry{builders: b} -} - -// URL types supported by the config store -const ( - // example fs:///tmp/testdata/configroot - FSUrl = "fs" -) - -// NewStore create a new store based on the config URL. -func (r *Registry) NewStore(configURL string) (KeyValueStore, error) { - u, err := url.Parse(configURL) - - if err != nil { - return nil, fmt.Errorf("invalid config URL %s %v", configURL, err) - } - - if u.Scheme == FSUrl { - return newFSStore(u.Path) - } - if builder, ok := r.builders[u.Scheme]; ok { - return builder(u) - } - - return nil, fmt.Errorf("unknown config URL %s %v", configURL, u) -} diff --git a/mixer/pkg/config/store/store2.go b/mixer/pkg/config/store/store2.go index a32de827030e..d02467378909 100644 --- a/mixer/pkg/config/store/store2.go +++ b/mixer/pkg/config/store/store2.go @@ -26,6 +26,16 @@ import ( "istio.io/istio/pkg/log" ) +// ChangeType denotes the type of a change +type ChangeType int + +const ( + // Update - change was an update or a create to a key. + Update ChangeType = iota + // Delete - key was removed. + Delete +) + // ErrNotFound is the error to be returned when the given key does not exist in the storage. var ErrNotFound = errors.New("not found") @@ -224,6 +234,12 @@ func NewRegistry2(inventory ...RegisterFunc2) *Registry2 { return &Registry2{builders: b} } +// URL types supported by the config store +const ( + // example fs:///tmp/testdata/configroot + FSUrl = "fs" +) + // NewStore2 creates a new Store2 instance with the specified backend. func (r *Registry2) NewStore2(configURL string) (Store2, error) { u, err := url.Parse(configURL) diff --git a/mixer/pkg/config/store/store_test.go b/mixer/pkg/config/store/store_test.go deleted file mode 100644 index 5708782f27ed..000000000000 --- a/mixer/pkg/config/store/store_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package store - -import ( - "errors" - "net/url" - "strings" - "testing" -) - -func testingRegister(m map[string]Builder) { - m["test"] = func(u *url.URL) (KeyValueStore, error) { - return nil, nil - } -} - -func TestNewStore(t *testing.T) { - r := NewRegistry(testingRegister) - for _, tt := range []struct { - url string - err error - }{ - {"fs:///tmp", nil}, - {"redis://:passwd@localhost:6379/1", errors.New("unknown")}, // redis module is not loaded - {"etcd:///tmp/testdata/configroot", errors.New("unknown")}, - {"/tmp/testdata/configroot", errors.New("unknown")}, - {"test:///test/url", nil}, - } { - t.Run(tt.url, func(t *testing.T) { - _, err := r.NewStore(tt.url) - if err == tt.err { - return - } - - if err != nil { - if tt.err == nil || !strings.Contains(err.Error(), tt.err.Error()) { - t.Errorf("got %s\nwant %s", err, tt.err) - } - } - }) - } -} diff --git a/mixer/pkg/server/server.go b/mixer/pkg/server/server.go index 46a78018b090..748c962dec81 100644 --- a/mixer/pkg/server/server.go +++ b/mixer/pkg/server/server.go @@ -103,7 +103,8 @@ func new(a *Args, p *patchTable) (*Server, error) { s.adapterGP = pool.NewGoroutinePool(adapterPoolSize, a.SingleThreaded) s.adapterGP.AddWorkers(adapterPoolSize) - adapterMap := config.InventoryMap(a.Adapters) + tmplRepo := template.NewRepository(a.Templates) + adapterMap := config.AdapterInfoMap(a.Adapters, tmplRepo.SupportsTemplate) // construct the gRPC options