add new config struct (arana-db#51)

pkg/config/config.go

@@ -153,6 +153,25 @@ func parse(path string) *Configuration {
 	return cfg
 }
 
+func parseV2(path string) *ConfigMap {
+	log.Infof("load config from :  %s", path)
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		log.Fatalf("[config] [default load] load config failed, error: %v", err)
+	}
+	cfg := &ConfigMap{}
+	if yamlFormat(path) {
+		err = yaml.Unmarshal(content, cfg)
+	} else {
+		err = errors.Errorf("[config] [default load] unsupport config file format")
+	}
+	if err != nil {
+		log.Fatalf("[config] [default load] json unmarshal config failed, error: %v", err)
+	}
+
+	return cfg
+}
+
 func yamlFormat(path string) bool {
 	ext := filepath.Ext(path)
 	if ext == ".yaml" || ext == ".yml" {
@@ -162,8 +181,16 @@ func yamlFormat(path string) bool {
 }
 
 // Load config file and parse
+// todo been deleted
 func Load(path string) *Configuration {
 	configPath, _ := filepath.Abs(path)
 	cfg := parse(configPath)
 	return cfg
 }
+
+// todo been renamed function name to Load
+func LoadV2(path string) *ConfigMap {
+	configPath, _ := filepath.Abs(path)
+	cfg := parseV2(configPath)
+	return cfg
+}

pkg/config/db.go

@@ -48,6 +48,98 @@ type (
 		IdleTimeout    time.Duration   `yaml:"idle_timeout" json:"-"`            // close backend direct connection after idle_timeout
 		IdleTimeoutStr string          `yaml:"-" json:"idle_timeout"`
 	}
+
+	ConfigMap struct {
+		TypeMeta
+		Metadata *ObjectMeta `yaml:"metadata" json:"metadata"`
+		Data     *ConfigData `validate:"required" yaml:"data" json:"data"`
+	}
+
+	TypeMeta struct {
+		Kind       string `yaml:"kind" json:"kind,omitempty"`
+		APIVersion string ` yaml:"apiVersion" json:"apiVersion,omitempty"`
+	}
+
+	ObjectMeta struct {
+		Name string `json:"name,omitempty" yaml:"apiVersion,omitempty"`
+	}
+
+	ConfigData struct {
+		// todo add Listeners and Executors config
+		// Listeners          []*Listener          `validate:"required" yaml:"listeners" json:"listeners"`
+		// Executors          []*Executor          `validate:"required" yaml:"executors" json:"executors"`
+		Filters            []string             `yaml:"filters" json:"filters"`
+		DataSourceClusters []*DataSourceCluster `validate:"required" yaml:"dataSourceClusters" json:"dataSourceClusters"`
+		ShardingRule       *ShardingRule        `yaml:"shardingRule,omitempty" json:"shardingRule,omitempty"`
+	}
+
+	DataSourceCluster struct {
+		Name        string         `yaml:"name" json:"name"`
+		Type        DataSourceType `yaml:"type" json:"type"`
+		SqlMaxLimit int            `default:"-1" yaml:"sqlMaxLimit" json:"sqlMaxLimit,omitempty"`
+		Tenant      string         `yaml:"tenant" json:"tenant"`
+		ConnProps   *ConnProp      `yaml:"connProps" json:"connProps,omitempty"`
+		Groups      []*Group       `yaml:"groups" json:"groups"`
+	}
+
+	ConnProp struct {
+		Capacity    int `yaml:"capacity" json:"capacity,omitempty"`       // connection pool capacity
+		MaxCapacity int `yaml:"maxCapacity" json:"maxCapacity,omitempty"` // max connection pool capacity
+		IdleTimeout int `yaml:"idleTimeout" json:"idleTimeout,omitempty"` // close backend direct connection after idle_timeout
+	}
+
+	Group struct {
+		Name    string `yaml:"name" json:"name"`
+		AtomDbs []*Dsn `yaml:"atomDbs" json:"atomDbs"`
+	}
+
+	Dsn struct {
+		Host     string         `yaml:"host" json:"host"`
+		Port     int            `yaml:"port" json:"port"`
+		Username string         `yaml:"username" json:"username"`
+		Password string         `yaml:"password" json:"password"`
+		Database string         `yaml:"database" json:"database"`
+		DsnProps *DsnConnProp   `yaml:"connProps" json:"connProps,omitempty"`
+		Role     DataSourceRole `yaml:"role" json:"role"`
+		Weight   string         `default:"r10w10" yaml:"weight" json:"weight"`
+	}
+
+	DsnConnProp struct {
+		ReadTimeout  string `yaml:"readTimeout" json:"readTimeout,omitempty"`
+		WriteTimeout string `yaml:"writeTimeout" json:"writeTimeout,omitempty"`
+		ParseTime    bool   `default:"true" yaml:"parseTime,omitempty" json:"parseTime,omitempty"`
+		Loc          string `yaml:"loc" json:"loc,omitempty"`
+		Charset      string `yaml:"charset" json:"charset,omitempty"`
+	}
+
+	ShardingRule struct {
+		Tables []*Table `yaml:"tables" json:"tables"`
+	}
+
+	Table struct {
+		Name            string    `yaml:"name" json:"name"`
+		AllowFullScan   bool      `yaml:"allowFullScan" json:"allowFullScan,omitempty"`
+		ActualDataNodes []string  `yaml:"actualDataNodes" json:"actualDataNodes"`
+		DbRules         []*Rule   `yaml:"dbRules" json:"dbRules"`
+		TblRules        []*Rule   `yaml:"tblRules" json:"tblRules"`
+		TblProps        *TblProps `yaml:"otherProps" json:"otherProps,omitempty"`
+		Topology        *Topology `yaml:"topology" json:"topology"`
+		ShadowTopology  *Topology `yaml:"shadowTopology" json:"shadowTopology"`
+	}
+
+	Rule struct {
+		Column string `yaml:"column" json:"column"`
+		Expr   string `yaml:"expr" json:"expr"`
+	}
+
+	TblProps struct {
+		SqlMaxLimit int `default:"-1" yaml:"sqlMaxLimit" json:"sqlMaxLimit"` // connection pool capacity
+	}
+
+	Topology struct {
+		DbPattern  string `yaml:"dbPattern" json:"dbPattern"`
+		TblPattern string `yaml:"tblPattern" json:"tblPattern"`
+	}
 )
 
 const (

pkg/config/db_test.go

@@ -0,0 +1,119 @@
+//
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package config
+
+import (
+	"testing"
+)
+
+import (
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMetadataConf(t *testing.T) {
+	configPath := "./testdata/config.yaml"
+	conf := LoadV2(configPath)
+	assert.NotNil(t, conf)
+
+	assert.Equal(t, "ConfigMap", conf.Kind)
+	assert.Equal(t, "1.0", conf.APIVersion)
+	assert.Equal(t, "arana-config", conf.Metadata.Name)
+}
+
+func TestListenersConf(t *testing.T) {
+	// todo
+}
+
+func TestExcutorsConf(t *testing.T) {
+	// todo
+}
+
+func TestFiltersConf(t *testing.T) {
+	// todo
+}
+
+func TestDataSourceClustersConf(t *testing.T) {
+	configPath := "./testdata/config.yaml"
+	conf := LoadV2(configPath)
+	assert.NotEqual(t, nil, conf)
+
+	assert.Equal(t, 1, len(conf.Data.DataSourceClusters))
+	dataSourceCluster := conf.Data.DataSourceClusters[0]
+	assert.Equal(t, "employee", dataSourceCluster.Name)
+	assert.Equal(t, DBMysql, dataSourceCluster.Type)
+	assert.Equal(t, -1, dataSourceCluster.SqlMaxLimit)
+	assert.Equal(t, "dksl", dataSourceCluster.Tenant)
+	assert.NotNil(t, dataSourceCluster.ConnProps)
+	assert.Equal(t, 20, dataSourceCluster.ConnProps.MaxCapacity)
+	assert.Equal(t, 60, dataSourceCluster.ConnProps.IdleTimeout)
+
+	assert.Equal(t, 1, len(dataSourceCluster.Groups))
+	group := dataSourceCluster.Groups[0]
+	assert.Equal(t, "employee_1", group.Name)
+	assert.Equal(t, 1, len(group.AtomDbs))
+	atomDb := group.AtomDbs[0]
+	assert.Equal(t, "127.0.0.1", atomDb.Host)
+	assert.Equal(t, 3306, atomDb.Port)
+	assert.Equal(t, "root", atomDb.Username)
+	assert.Equal(t, "123456", atomDb.Password)
+	assert.Equal(t, "employees_0001", atomDb.Database)
+	assert.Equal(t, Master, atomDb.Role)
+	assert.Equal(t, "r0w10", atomDb.Weight)
+	assert.NotNil(t, atomDb.DsnProps)
+	assert.Equal(t, "1s", atomDb.DsnProps.ReadTimeout)
+	assert.Equal(t, "1s", atomDb.DsnProps.WriteTimeout)
+	assert.Equal(t, true, atomDb.DsnProps.ParseTime)
+	assert.Equal(t, "Local", atomDb.DsnProps.Loc)
+	assert.Equal(t, "utf8mb4,utf8", atomDb.DsnProps.Charset)
+}
+
+func TestShardingRuleConf(t *testing.T) {
+	configPath := "./testdata/config.yaml"
+	conf := LoadV2(configPath)
+	assert.NotEqual(t, nil, conf)
+
+	assert.NotNil(t, conf.Data.ShardingRule)
+	assert.Equal(t, 1, len(conf.Data.ShardingRule.Tables))
+	table := conf.Data.ShardingRule.Tables[0]
+	assert.Equal(t, table.Name, "employee.student")
+	assert.Equal(t, table.AllowFullScan, true)
+
+	assert.Equal(t, 2, len(table.ActualDataNodes))
+	assert.Equal(t, "employee_tmall.student", table.ActualDataNodes[0])
+	assert.Equal(t, "employee_taobao.student", table.ActualDataNodes[1])
+
+	assert.Equal(t, 2, len(table.DbRules))
+	assert.Equal(t, "student_id", table.DbRules[0].Column)
+	assert.Equal(t, "modShard(3)", table.DbRules[0].Expr)
+	assert.Equal(t, "student_num", table.DbRules[1].Column)
+	assert.Equal(t, "hashMd5Shard(3)", table.DbRules[1].Expr)
+
+	assert.Equal(t, 1, len(table.TblRules))
+	assert.Equal(t, "student_id", table.TblRules[0].Column)
+	assert.Equal(t, "modShard(3)", table.TblRules[0].Expr)
+
+	assert.NotNil(t, table.TblProps)
+	assert.Equal(t, -1, table.TblProps.SqlMaxLimit)
+
+	assert.Equal(t, "employee_${0000...0002}", table.Topology.DbPattern)
+	assert.Equal(t, "student_${0000...0008}", table.Topology.TblPattern)
+	assert.Equal(t, "employee_${0000...0002}", table.ShadowTopology.DbPattern)
+	assert.Equal(t, "student_${0000...0008}", table.ShadowTopology.TblPattern)
+}

pkg/config/testdata/config.yaml

@@ -0,0 +1,70 @@
+kind: ConfigMap
+apiVersion: "1.0"
+metadata:
+  name: arana-config
+data:
+  listeners:
+    - protocol_type: mysql
+      socket_address:
+        address: 0.0.0.0
+        port: 13306
+      config:
+        users:
+          dksl: "123456"
+        server_version: 5.7.0
+      executor: redirect
+
+  executors:
+    - name: redirect
+      mode: singledb
+      data_sources:
+        - master: employees
+
+  dataSourceClusters:
+    - name: employee
+      type: mysql
+      sqlMaxLimit: -1
+      tenant: dksl
+      connProps:
+        capacity: 10
+        maxCapacity: 20
+        idleTimeout: 60
+      groups:
+        - name: employee_1
+          atomDbs:
+            - host: 127.0.0.1
+              port: 3306
+              username: root
+              password: "123456"
+              database: employees_0001
+              role: master
+              weight: r0w10
+              connProps:
+                readTimeout: "1s"
+                writeTimeout: "1s"
+                parseTime: true
+                loc: Local
+                charset: utf8mb4,utf8
+  shardingRule:
+    tables:
+      - name: employee.student
+        allowFullScan: true
+        actualDataNodes:
+          - employee_tmall.student
+          - employee_taobao.student
+        dbRules:
+          - column: student_id
+            expr: modShard(3)
+          - column: student_num
+            expr: hashMd5Shard(3)
+        tblRules:
+          - column: student_id
+            expr: modShard(3)
+        otherProps:
+          sqlMaxLimit: -1
+        topology:
+          dbPattern: employee_${0000...0002}
+          tblPattern: student_${0000...0008}
+        shadowTopology:
+          dbPattern: employee_${0000...0002}
+          tblPattern: student_${0000...0008}