Deep merge config files (cadence-workflow#4165)

* Deep merge config files * Updated change log * Updated config for scylla
diagnotes · Jul 9, 2021 · c185ad8 · c185ad8
1 parent d91e86f
commit c185ad8
Show file tree

Hide file tree

Showing 11 changed files with 56 additions and 671 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,6 +18,7 @@ You can find a list of previous releases on the [github releases](https://github
 - Change `--connect-attributes` in `cadence-sql-tool` from URL encoding to the format of k1=v1,k2=v2...
 - Change `--domain_data` in `cadence domain update/register` from the format of k1:v1,k2:v2... to the format of k1=v1,k2=v2...
 - Deprecate local domains. All new domains should be created as global domains.
+- Server will deep merge configuration files when loading. No need to copy the whole config section, specify only those fields that needs overriding. (#4165)
 
 ## [0.18.0] - 2021-01-22
 

diff --git a/common/config/loader.go b/common/config/loader.go
@@ -22,12 +22,11 @@ package config
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 
+	uconfig "go.uber.org/config"
 	"gopkg.in/validator.v2"
-	"gopkg.in/yaml.v2"
 )
 
 const (
@@ -80,17 +79,18 @@ func Load(env string, configDir string, zone string, config interface{}) error {
 
 	log.Printf("Loading configFiles=%v\n", files)
 
+	var options []uconfig.YAMLOption
 	for _, f := range files {
-		// This is tagged nosec because the file names being read are for config files that are not user supplied
-		// #nosec
-		data, err := ioutil.ReadFile(f)
-		if err != nil {
-			return err
-		}
-		err = yaml.Unmarshal(data, config)
-		if err != nil {
-			return err
-		}
+		options = append(options, uconfig.File(f))
+	}
+	yaml, err := uconfig.NewYAML(options...)
+	if err != nil {
+		return err
+	}
+
+	err = yaml.Get(uconfig.Root).Populate(config)
+	if err != nil {
+		return err
 	}
 
 	return validator.Validate(config)

diff --git a/common/config/loader_test.go b/common/config/loader_test.go
@@ -59,9 +59,10 @@ func (s *LoaderSuite) TestBaseYaml() {
 	s.Nil(err)
 	defer os.RemoveAll(dir)
 
-	data := buildConfig("", "")
-	err = ioutil.WriteFile(path(dir, "base.yaml"), []byte(data), fileMode)
-	s.Nil(err)
+	s.createFile(dir, "base.yaml", `
+items:
+  item1: base1
+  item2: base2`)
 
 	envs := []string{"", "prod"}
 	zones := []string{"", "us-east-1a"}
@@ -71,8 +72,8 @@ func (s *LoaderSuite) TestBaseYaml() {
 			var cfg testConfig
 			err = Load(env, dir, zone, &cfg)
 			s.Nil(err)
-			s.Equal("hello__", cfg.Items.Item1)
-			s.Equal("world__", cfg.Items.Item2)
+			s.Equal("base1", cfg.Items.Item1)
+			s.Equal("base2", cfg.Items.Item2)
 		}
 	}
 }
@@ -83,26 +84,38 @@ func (s *LoaderSuite) TestHierarchy() {
 	s.Nil(err)
 	defer os.RemoveAll(dir)
 
-	s.createFile(dir, "base.yaml", "", "")
-	s.createFile(dir, "development.yaml", "development", "")
-	s.createFile(dir, "prod.yaml", "prod", "")
-	s.createFile(dir, "prod_dca.yaml", "prod", "dca")
+	s.createFile(dir, "base.yaml", `
+items:
+  item1: base1
+  item2: base2`)
+
+	s.createFile(dir, "development.yaml", `
+items:
+  item1: development1`)
+
+	s.createFile(dir, "prod.yaml", `
+items:
+  item1: prod1`)
+
+	s.createFile(dir, "prod_dca.yaml", `
+items:
+  item1: prod_dca1`)
 
 	testCases := []struct {
 		env   string
 		zone  string
 		item1 string
 		item2 string
 	}{
-		{"", "", "hello_development_", "world_development_"},
-		{"", "dca", "hello_development_", "world_development_"},
-		{"", "pdx", "hello_development_", "world_development_"},
-		{"development", "", "hello_development_", "world_development_"},
-		{"development", "dca", "hello_development_", "world_development_"},
-		{"development", "pdx", "hello_development_", "world_development_"},
-		{"prod", "", "hello_prod_", "world_prod_"},
-		{"prod", "dca", "hello_prod_dca", "world_prod_dca"},
-		{"prod", "pdx", "hello_prod_", "world_prod_"},
+		{"", "", "development1", "base2"},
+		{"", "dca", "development1", "base2"},
+		{"", "pdx", "development1", "base2"},
+		{"development", "", "development1", "base2"},
+		{"development", "dca", "development1", "base2"},
+		{"development", "pdx", "development1", "base2"},
+		{"prod", "", "prod1", "base2"},
+		{"prod", "dca", "prod_dca1", "base2"},
+		{"prod", "pdx", "prod1", "base2"},
 	}
 
 	for _, tc := range testCases {
@@ -120,16 +133,7 @@ func (s *LoaderSuite) TestInvalidPath() {
 	s.NotNil(err)
 }
 
-func (s *LoaderSuite) createFile(dir string, file string, env string, zone string) {
-	err := ioutil.WriteFile(path(dir, file), []byte(buildConfig(env, zone)), fileMode)
+func (s *LoaderSuite) createFile(dir string, file string, content string) {
+	err := ioutil.WriteFile(path(dir, file), []byte(content), fileMode)
 	s.Nil(err)
 }
-
-func buildConfig(env, zone string) string {
-	item1 := concat("hello", concat(env, zone))
-	item2 := concat("world", concat(env, zone))
-	return `
-    items:
-      item1: ` + item1 + `
-      item2: ` + item2
-}
diff --git a/config/development_es.yaml b/config/development_es.yaml
@@ -1,13 +1,6 @@
 persistence:
-  defaultStore: cass-default
   advancedVisibilityStore: es-visibility
-  numHistoryShards: 4
   datastores:
-    cass-default:
-      nosql:
-        pluginName: "cassandra"
-        hosts: "127.0.0.1"
-        keyspace: "cadence"
     es-visibility:
       elasticsearch:
         url:
@@ -16,101 +9,6 @@ persistence:
         indices:
           visibility: cadence-visibility-dev
 
-ringpop:
-  name: cadence
-  bootstrapMode: hosts
-  bootstrapHosts: ["127.0.0.1:7933", "127.0.0.1:7934", "127.0.0.1:7935"]
-  maxJoinDuration: 30s
-
-services:
-  frontend:
-    rpc:
-      port: 7933
-      grpcPort: 7833
-      bindOnLocalHost: true
-    metrics:
-      statsd:
-        hostPort: "127.0.0.1:8125"
-        prefix: "cadence"
-    pprof:
-      port: 7936
-
-  matching:
-    rpc:
-      port: 7935
-      grpcPort: 7835
-      bindOnLocalHost: true
-    metrics:
-      statsd:
-        hostPort: "127.0.0.1:8125"
-        prefix: "cadence"
-    pprof:
-      port: 7938
-
-  history:
-    rpc:
-      port: 7934
-      grpcPort: 7834
-      bindOnLocalHost: true
-    metrics:
-      statsd:
-        hostPort: "127.0.0.1:8125"
-        prefix: "cadence"
-    pprof:
-      port: 7937
-
-  worker:
-    rpc:
-      port: 7939
-      bindOnLocalHost: true
-    metrics:
-      statsd:
-        hostPort: "127.0.0.1:8125"
-        prefix: "cadence"
-    pprof:
-      port: 7940
-
-clusterMetadata:
-  enableGlobalDomain: true
-  failoverVersionIncrement: 10
-  primaryClusterName: "active"
-  currentClusterName: "active"
-  clusterInformation:
-    active:
-      enabled: true
-      initialFailoverVersion: 0
-      rpcName: "cadence-frontend"
-      rpcAddress: "localhost:7933"
-
-dcRedirectionPolicy:
-  policy: "noop"
-  toDC: ""
-
-archival:
-  history:
-    status: "enabled"
-    enableRead: true
-    provider:
-      filestore:
-        fileMode: "0666"
-        dirMode: "0766"
-  visibility:
-    status: "enabled"
-    enableRead: true
-    provider:
-      filestore:
-        fileMode: "0666"
-        dirMode: "0766"
-
-domainDefaults:
-  archival:
-    history:
-      status: "enabled"
-      URI: "file:///tmp/cadence_archival/development"
-    visibility:
-      status: "enabled"
-      URI: "file:///tmp/cadence_vis_archival/development"
-
 kafka:
   tls:
     enabled: false
@@ -128,13 +26,5 @@ kafka:
       topic: cadence-visibility-dev
       dlq-topic: cadence-visibility-dev-dlq
 
-publicClient:
-  hostPort: "localhost:7933"
-
 dynamicConfigClient:
   filepath: "config/dynamicconfig/development_es.yaml"
-  pollInterval: "10s"
-
-blobstore:
-  filestore:
-    outputDirectory: "/tmp/blobstore"