configmap: Add priorities to configmap Setters rclone#5178

fs/config/configmap/configmap.go

@@ -6,6 +6,17 @@ import (
 	"strings"
 )
 
+// Priority of getters
+type Priority int8
+
+// Priority levels for AddGetter
+const (
+	PriorityNormal  Priority = iota
+	PriorityConfig           // use for reading from the config
+	PriorityDefault          // use for default values
+	PriorityMax
+)
+
 // Getter provides an interface to get config items
 type Getter interface {
 	// Get should get an item with the key passed in and return
@@ -29,28 +40,26 @@ type Mapper interface {
 // Map provides a wrapper around multiple Setter and
 // Getter interfaces.
 type Map struct {
-	setters  []Setter
-	getters  []Getter
-	override []Getter
+	setters []Setter
+	getters []getprio
+}
+
+type getprio struct {
+	getter   Getter
+	priority Priority
 }
 
 // New returns an empty Map
 func New() *Map {
 	return &Map{}
 }
 
-// AddGetter appends a getter onto the end of the getters
-func (c *Map) AddGetter(getter Getter) *Map {
-	c.getters = append(c.getters, getter)
-	return c
-}
-
-// AddOverrideGetter appends a getter onto the end of the getters
-//
-// It also appends it onto the override getters for GetOverride
-func (c *Map) AddOverrideGetter(getter Getter) *Map {
-	c.getters = append(c.getters, getter)
-	c.override = append(c.override, getter)
+// AddGetter appends a getter onto the end of the getters in priority order
+func (c *Map) AddGetter(getter Getter, priority Priority) *Map {
+	c.getters = append(c.getters, getprio{getter, priority})
+	sort.SliceStable(c.getters, func(i, j int) bool {
+		return c.getters[i].priority < c.getters[j].priority
+	})
 	return c
 }
 
@@ -66,12 +75,28 @@ func (c *Map) ClearSetters() *Map {
 	return c
 }
 
-// get gets an item with the key passed in and return the value from
-// the first getter. If the item is found then it returns true,
-// otherwise false.
-func (c *Map) get(key string, getters []Getter) (value string, ok bool) {
-	for _, do := range getters {
-		value, ok = do.Get(key)
+// ClearGetters removes all the getters with the priority given
+func (c *Map) ClearGetters(priority Priority) *Map {
+	getters := c.getters[:0]
+	for _, item := range c.getters {
+		if item.priority != priority {
+			getters = append(getters, item)
+		}
+	}
+	c.getters = getters
+	return c
+}
+
+// GetPriority gets an item with the key passed in and return the
+// value from the first getter to return a result with priority <=
+// maxPriority. If the item is found then it returns true, otherwise
+// false.
+func (c *Map) GetPriority(key string, maxPriority Priority) (value string, ok bool) {
+	for _, item := range c.getters {
+		if item.priority > maxPriority {
+			break
+		}
+		value, ok = item.getter.Get(key)
 		if ok {
 			return value, ok
 		}
@@ -83,14 +108,7 @@ func (c *Map) get(key string, getters []Getter) (value string, ok bool) {
 // the first getter. If the item is found then it returns true,
 // otherwise false.
 func (c *Map) Get(key string) (value string, ok bool) {
-	return c.get(key, c.getters)
-}
-
-// GetOverride gets an item with the key passed in and return the
-// value from the first override getter. If the item is found then it
-// returns true, otherwise false.
-func (c *Map) GetOverride(key string) (value string, ok bool) {
-	return c.get(key, c.override)
+	return c.GetPriority(key, PriorityMax)
 }
 
 // Set sets an item into all the stored setters.

fs/config/configmap/configmap_test.go

@@ -27,7 +27,7 @@ func TestConfigMapGet(t *testing.T) {
 		"config1": "one",
 	}
 
-	m.AddGetter(m1)
+	m.AddGetter(m1, PriorityNormal)
 
 	value, found = m.Get("config1")
 	assert.Equal(t, "one", value)
@@ -42,7 +42,7 @@ func TestConfigMapGet(t *testing.T) {
 		"config2": "two2",
 	}
 
-	m.AddGetter(m2)
+	m.AddGetter(m2, PriorityNormal)
 
 	value, found = m.Get("config1")
 	assert.Equal(t, "one", value)
@@ -105,54 +105,143 @@ func TestConfigMapSet(t *testing.T) {
 
 }
 
-func TestConfigMapGetOverride(t *testing.T) {
+func TestConfigMapGetPriority(t *testing.T) {
 	m := New()
 
-	value, found := m.GetOverride("config1")
+	value, found := m.GetPriority("config1", PriorityMax)
 	assert.Equal(t, "", value)
 	assert.Equal(t, false, found)
 
-	value, found = m.GetOverride("config2")
+	value, found = m.GetPriority("config2", PriorityMax)
 	assert.Equal(t, "", value)
 	assert.Equal(t, false, found)
 
 	m1 := Simple{
 		"config1": "one",
+		"config3": "three",
 	}
 
-	m.AddOverrideGetter(m1)
+	m.AddGetter(m1, PriorityConfig)
+
+	value, found = m.GetPriority("config1", PriorityNormal)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, found)
 
-	value, found = m.GetOverride("config1")
+	value, found = m.GetPriority("config2", PriorityNormal)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, found)
+
+	value, found = m.GetPriority("config3", PriorityNormal)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, found)
+
+	value, found = m.GetPriority("config1", PriorityConfig)
 	assert.Equal(t, "one", value)
 	assert.Equal(t, true, found)
 
-	value, found = m.GetOverride("config2")
+	value, found = m.GetPriority("config2", PriorityConfig)
 	assert.Equal(t, "", value)
 	assert.Equal(t, false, found)
 
+	value, found = m.GetPriority("config3", PriorityConfig)
+	assert.Equal(t, "three", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config1", PriorityMax)
+	assert.Equal(t, "one", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config2", PriorityMax)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, found)
+
+	value, found = m.GetPriority("config3", PriorityMax)
+	assert.Equal(t, "three", value)
+	assert.Equal(t, true, found)
+
 	m2 := Simple{
 		"config1": "one2",
 		"config2": "two2",
 	}
 
-	m.AddGetter(m2)
+	m.AddGetter(m2, PriorityNormal)
 
-	value, found = m.GetOverride("config1")
-	assert.Equal(t, "one", value)
+	value, found = m.GetPriority("config1", PriorityNormal)
+	assert.Equal(t, "one2", value)
 	assert.Equal(t, true, found)
 
-	value, found = m.GetOverride("config2")
+	value, found = m.GetPriority("config2", PriorityNormal)
+	assert.Equal(t, "two2", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config3", PriorityNormal)
 	assert.Equal(t, "", value)
 	assert.Equal(t, false, found)
 
-	value, found = m.Get("config1")
-	assert.Equal(t, "one", value)
+	value, found = m.GetPriority("config1", PriorityConfig)
+	assert.Equal(t, "one2", value)
 	assert.Equal(t, true, found)
 
-	value, found = m.Get("config2")
+	value, found = m.GetPriority("config2", PriorityConfig)
 	assert.Equal(t, "two2", value)
 	assert.Equal(t, true, found)
 
+	value, found = m.GetPriority("config3", PriorityConfig)
+	assert.Equal(t, "three", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config1", PriorityMax)
+	assert.Equal(t, "one2", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config2", PriorityMax)
+	assert.Equal(t, "two2", value)
+	assert.Equal(t, true, found)
+
+	value, found = m.GetPriority("config3", PriorityMax)
+	assert.Equal(t, "three", value)
+	assert.Equal(t, true, found)
+}
+
+func TestConfigMapClearGetters(t *testing.T) {
+	m := New()
+	m1 := Simple{}
+	m2 := Simple{}
+	m3 := Simple{}
+	m.AddGetter(m1, PriorityNormal)
+	m.AddGetter(m2, PriorityDefault)
+	m.AddGetter(m3, PriorityConfig)
+	assert.Equal(t, []getprio{
+		{m1, PriorityNormal},
+		{m3, PriorityConfig},
+		{m2, PriorityDefault},
+	}, m.getters)
+	m.ClearGetters(PriorityConfig)
+	assert.Equal(t, []getprio{
+		{m1, PriorityNormal},
+		{m2, PriorityDefault},
+	}, m.getters)
+	m.ClearGetters(PriorityNormal)
+	assert.Equal(t, []getprio{
+		{m2, PriorityDefault},
+	}, m.getters)
+	m.ClearGetters(PriorityDefault)
+	assert.Equal(t, []getprio{}, m.getters)
+	m.ClearGetters(PriorityDefault)
+	assert.Equal(t, []getprio{}, m.getters)
+}
+
+func TestConfigMapClearSetters(t *testing.T) {
+	m := New()
+	m1 := Simple{}
+	m2 := Simple{}
+	m3 := Simple{}
+	m.AddSetter(m1)
+	m.AddSetter(m2)
+	m.AddSetter(m3)
+	assert.Equal(t, []Setter{m1, m2, m3}, m.setters)
+	m.ClearSetters()
+	assert.Equal(t, []Setter(nil), m.setters)
 }
 
 func TestSimpleString(t *testing.T) {

fs/fs.go

@@ -132,7 +132,7 @@ func (os Options) Overridden(m *configmap.Map) configmap.Simple {
 	var overridden = configmap.Simple{}
 	for i := range os {
 		opt := &os[i]
-		value, isSet := m.GetOverride(opt.Name)
+		value, isSet := m.GetPriority(opt.Name, configmap.PriorityNormal)
 		if isSet {
 			overridden.Set(opt.Name, value)
 		}
@@ -1336,28 +1336,28 @@ func ConfigMap(fsInfo *RegInfo, configName string, connectionStringConfig config
 
 	// Config from connection string
 	if len(connectionStringConfig) > 0 {
-		config.AddOverrideGetter(connectionStringConfig)
+		config.AddGetter(connectionStringConfig, configmap.PriorityNormal)
 	}
 
 	// flag values
 	if fsInfo != nil {
-		config.AddOverrideGetter(&regInfoValues{fsInfo, false})
+		config.AddGetter(&regInfoValues{fsInfo, false}, configmap.PriorityNormal)
 	}
 
 	// remote specific environment vars
-	config.AddOverrideGetter(configEnvVars(configName))
+	config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)
 
 	// backend specific environment vars
 	if fsInfo != nil {
-		config.AddOverrideGetter(optionEnvVars{fsInfo: fsInfo})
+		config.AddGetter(optionEnvVars{fsInfo: fsInfo}, configmap.PriorityNormal)
 	}
 
 	// config file
-	config.AddGetter(getConfigFile(configName))
+	config.AddGetter(getConfigFile(configName), configmap.PriorityConfig)
 
 	// default values
 	if fsInfo != nil {
-		config.AddGetter(&regInfoValues{fsInfo, true})
+		config.AddGetter(&regInfoValues{fsInfo, true}, configmap.PriorityDefault)
 	}
 
 	// Set Config