Custom data (easegress-io#500)

* add dynamic object * add custom data store * update custom data API * update API for mesh * add client command for custom data * fix mesh layout typo * rename for adding test cases * add unit tests * add more testcases * improve coverage * add documentation for custom data * Apply suggestions from code review Co-authored-by: Samu Tamminen <[email protected]> * Apply suggestions from code review Co-authored-by: Samu Tamminen <[email protected]> * update according to comment Co-authored-by: Samu Tamminen <[email protected]>
xixisidi · Feb 11, 2022 · e2a96b8 · e2a96b8
1 parent a99ec2d
commit e2a96b8
Show file tree

Hide file tree

Showing 18 changed files with 1,897 additions and 299 deletions.
diff --git a/cmd/client/command/common.go b/cmd/client/command/common.go
@@ -63,8 +63,10 @@ const (
 	wasmCodeURL = apiURL + "/wasm/code"
 	wasmDataURL = apiURL + "/wasm/data/%s/%s"
 
-	customDataKindURL = apiURL + "/customdata/%s"
-	customDataURL     = apiURL + "/customdata/%s/%s"
+	customDataKindURL     = apiURL + "/customdatakinds"
+	customDataKindItemURL = apiURL + "/customdatakinds/%s"
+	customDataURL         = apiURL + "/customdata/%s"
+	customDataItemURL     = apiURL + "/customdata/%s/%s"
 
 	// MeshTenantsURL is the mesh tenant prefix.
 	MeshTenantsURL = apiURL + "/mesh/tenants"

diff --git a/cmd/client/command/customdata.go b/cmd/client/command/customdata.go
@@ -24,6 +24,118 @@ import (
 	"github.com/spf13/cobra"
 )
 
+// CustomDataKindCmd defines custom data kind command.
+func CustomDataKindCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "custom-data-kind",
+		Short: "View and change custom data kind",
+	}
+
+	cmd.AddCommand(listCustomDataKindCmd())
+	cmd.AddCommand(getCustomDataKindCmd())
+	cmd.AddCommand(createCustomDataKindCmd())
+	cmd.AddCommand(updateCustomDataKindCmd())
+	cmd.AddCommand(deleteCustomDataKindCmd())
+
+	return cmd
+}
+
+func listCustomDataKindCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "list",
+		Short:   "List all custom data kinds",
+		Example: "egctl custom-data-kind list",
+
+		Run: func(cmd *cobra.Command, args []string) {
+			handleRequest(http.MethodGet, makeURL(customDataKindURL), nil, cmd)
+		},
+	}
+
+	return cmd
+}
+
+func getCustomDataKindCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "get",
+		Short:   "Get a custom data kind",
+		Example: "egctl custom-data-kind get <kind>",
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return errors.New("requires custom data kind to be retrieved")
+			}
+			return nil
+		},
+
+		Run: func(cmd *cobra.Command, args []string) {
+			handleRequest(http.MethodGet, makeURL(customDataKindItemURL, args[0]), nil, cmd)
+		},
+	}
+
+	return cmd
+}
+
+func createCustomDataKindCmd() *cobra.Command {
+	var specFile string
+	cmd := &cobra.Command{
+		Use:     "create",
+		Short:   "Create a custom data kind from a yaml file or stdin",
+		Example: "egctl custom-data-kind create -f <kind file>",
+		Run: func(cmd *cobra.Command, args []string) {
+			visitor := buildYAMLVisitor(specFile, cmd)
+			visitor.Visit(func(yamlDoc []byte) error {
+				handleRequest(http.MethodPost, makeURL(customDataKindURL), yamlDoc, cmd)
+				return nil
+			})
+			visitor.Close()
+		},
+	}
+
+	cmd.Flags().StringVarP(&specFile, "file", "f", "", "A yaml file specifying the change request.")
+
+	return cmd
+}
+
+func updateCustomDataKindCmd() *cobra.Command {
+	var specFile string
+	cmd := &cobra.Command{
+		Use:     "update",
+		Short:   "Update a custom data from a yaml file or stdin",
+		Example: "egctl custom-data-kind update -f <kind file>",
+		Run: func(cmd *cobra.Command, args []string) {
+			visitor := buildYAMLVisitor(specFile, cmd)
+			visitor.Visit(func(yamlDoc []byte) error {
+				handleRequest(http.MethodPut, makeURL(customDataKindURL), yamlDoc, cmd)
+				return nil
+			})
+			visitor.Close()
+		},
+	}
+
+	cmd.Flags().StringVarP(&specFile, "file", "f", "", "A yaml file specifying the change request.")
+
+	return cmd
+}
+
+func deleteCustomDataKindCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "delete",
+		Short:   "Delete a custom data kind",
+		Example: "egctl custom-data-kind delete <kind>",
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return errors.New("requires custom data kind to be retrieved")
+			}
+			return nil
+		},
+
+		Run: func(cmd *cobra.Command, args []string) {
+			handleRequest(http.MethodDelete, makeURL(customDataKindItemURL, args[0]), nil, cmd)
+		},
+	}
+
+	return cmd
+}
+
 // CustomDataCmd defines custom data command.
 func CustomDataCmd() *cobra.Command {
 	cmd := &cobra.Command{
@@ -33,15 +145,18 @@ func CustomDataCmd() *cobra.Command {
 
 	cmd.AddCommand(listCustomDataCmd())
 	cmd.AddCommand(getCustomDataCmd())
+	cmd.AddCommand(createCustomDataCmd())
 	cmd.AddCommand(updateCustomDataCmd())
+	cmd.AddCommand(batchUpdateCustomDataCmd())
+	cmd.AddCommand(deleteCustomDataCmd())
 
 	return cmd
 }
 
 func getCustomDataCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "get",
-		Short:   "Get an custom data",
+		Short:   "Get a custom data",
 		Example: "egctl custom-data get <kind> <id>",
 		Args: func(cmd *cobra.Command, args []string) error {
 			if len(args) != 2 {
@@ -51,7 +166,7 @@ func getCustomDataCmd() *cobra.Command {
 		},
 
 		Run: func(cmd *cobra.Command, args []string) {
-			handleRequest(http.MethodGet, makeURL(customDataURL, args[0], args[1]), nil, cmd)
+			handleRequest(http.MethodGet, makeURL(customDataItemURL, args[0], args[1]), nil, cmd)
 		},
 	}
 
@@ -70,19 +185,73 @@ func listCustomDataCmd() *cobra.Command {
 			return nil
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			handleRequest(http.MethodGet, makeURL(customDataKindURL, args[0]), nil, cmd)
+			handleRequest(http.MethodGet, makeURL(customDataURL, args[0]), nil, cmd)
 		},
 	}
 
 	return cmd
 }
 
+func createCustomDataCmd() *cobra.Command {
+	var specFile string
+	cmd := &cobra.Command{
+		Use:     "create",
+		Short:   "Create a custom data from a yaml file or stdin",
+		Example: "egctl custom-data create <kind> -f <data item file>",
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return errors.New("requires custom data kind to be retrieved")
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			visitor := buildYAMLVisitor(specFile, cmd)
+			visitor.Visit(func(yamlDoc []byte) error {
+				handleRequest(http.MethodPost, makeURL(customDataURL, args[0]), yamlDoc, cmd)
+				return nil
+			})
+			visitor.Close()
+		},
+	}
+
+	cmd.Flags().StringVarP(&specFile, "file", "f", "", "A yaml file specifying the change request.")
+
+	return cmd
+}
+
 func updateCustomDataCmd() *cobra.Command {
 	var specFile string
 	cmd := &cobra.Command{
 		Use:     "update",
+		Short:   "Update a custom data from a yaml file or stdin",
+		Example: "egctl custom-data update <kind> -f <data item file>",
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return errors.New("requires custom data kind to be retrieved")
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			visitor := buildYAMLVisitor(specFile, cmd)
+			visitor.Visit(func(yamlDoc []byte) error {
+				handleRequest(http.MethodPut, makeURL(customDataURL, args[0]), yamlDoc, cmd)
+				return nil
+			})
+			visitor.Close()
+		},
+	}
+
+	cmd.Flags().StringVarP(&specFile, "file", "f", "", "A yaml file specifying the change request.")
+
+	return cmd
+}
+
+func batchUpdateCustomDataCmd() *cobra.Command {
+	var specFile string
+	cmd := &cobra.Command{
+		Use:     "batch-update",
 		Short:   "Batch update custom data from a yaml file or stdin",
-		Example: "egctl custom-data update <kind> -f <change request file>",
+		Example: "egctl custom-data batch-update <kind> -f <change request file>",
 		Args: func(cmd *cobra.Command, args []string) error {
 			if len(args) != 1 {
 				return errors.New("requires custom data kind to be retrieved")
@@ -92,7 +261,7 @@ func updateCustomDataCmd() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			visitor := buildYAMLVisitor(specFile, cmd)
 			visitor.Visit(func(yamlDoc []byte) error {
-				handleRequest(http.MethodPost, makeURL(customDataKindURL, args[0]), yamlDoc, cmd)
+				handleRequest(http.MethodPost, makeURL(customDataItemURL, args[0], "items"), yamlDoc, cmd)
 				return nil
 			})
 			visitor.Close()
@@ -103,3 +272,22 @@ func updateCustomDataCmd() *cobra.Command {
 
 	return cmd
 }
+
+func deleteCustomDataCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "delete",
+		Short:   "Delete a custom data item",
+		Example: "egctl custom-data delete <kind> <id>",
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 2 {
+				return errors.New("requires custom data kind and id to be retrieved")
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			handleRequest(http.MethodDelete, makeURL(customDataItemURL, args[0], args[1]), nil, cmd)
+		},
+	}
+
+	return cmd
+}
diff --git a/cmd/client/main.go b/cmd/client/main.go
@@ -110,6 +110,7 @@ func main() {
 		command.ObjectCmd(),
 		command.MemberCmd(),
 		command.WasmCmd(),
+		command.CustomDataKindCmd(),
 		command.CustomDataCmd(),
 		completionCmd,
 	)

diff --git a/doc/README.md b/doc/README.md
@@ -9,6 +9,7 @@
 			- [4.1.1 System Controllers](#411-system-controllers)
 			- [4.1.2 Business Controllers](#412-business-controllers)
 		- [4.2 Filters](#42-filters)
+		- [4.3 Custom Data](#43-custom-data)
 
 ## 1. Cookbook / How-To Guide
 
@@ -91,4 +92,6 @@ It could be created, updated, deleted by admin operation. They control various r
 - [Validator](./reference/filters.md#Validator) - The Validator filter validates requests, forwards valid ones, and rejects invalid ones. 
 - [WasmHost](./reference/filters.md#WasmHost) - The WasmHost filter implements a host environment for user-developed WebAssembly code. 
 
+### 4.3 Custom Data
 
+- [Custom Data Management](./reference/customdata.md) - Create/Read/Update/Delete custom data kinds and custom data items.
diff --git a/doc/reference/customdata.md b/doc/reference/customdata.md
@@ -0,0 +1,105 @@
+# Custom Data Management
+
+The `Custom Data` feature implements a storage for 'any' data, which can be used by other components for data persistence.
+
+Because the schema of the data being persisted varies from components, a `CustomDataKind` must be defined to distinguish the data.
+
+## CustomDataKind
+
+The YAML example below defines a CustomDataKind:
+
+```yaml
+name: kind1
+idField: name
+jsonSchema:
+  type: object
+  properties:
+    name:
+      type: string
+      required: true
+```
+
+The `name` field is required, it is the kind name of custom data items of this kind.
+The `idField` is optional, and its default value is `name`, the value of this field of a data item is used as its identifier.
+The `jsonSchema` is optional, if provided, data items of this kind will be validated against this [JSON Schema](http://json-schema.org/).
+
+## CustomData
+
+CustomData is a map, the keys of this map must be strings while the values can be any valid JSON values, but the keys of a nested map must be strings too.
+
+A CustomData item must contain the `idField` defined by its corresponding CustomDataKind, for example, the data items of the kind defined in the above example must contain the `name` field as their identifiers.
+
+Below is an example of a CustomData:
+
+```yaml
+name: data1
+field1: 12
+field2: abc
+field3: [1, 2, 3, 4]
+```
+
+## API
+
+* **Create a CustomDataKind**
+	* **URL**: http://{ip}:{port}/apis/v1/customdatakinds
+	* **Method**: POST
+	* **Body**: CustomDataKind definition is YAML.
+
+* **Update a CustomDataKind**
+	* **URL**: http://{ip}:{port}/apis/v1/customdatakinds
+	* **Method**: PUT
+	* **Body**: CustomDataKind definition is YAML.
+
+* **Query the definition of a CustomDataKind**
+	* **URL**: http://{ip}:{port}/apis/v1/customdatakinds/{kind name}
+	* **Method**: GET
+
+* **List the definition of all CustomDataKind**
+	* **URL**: http://{ip}:{port}/apis/v1/customdatakinds
+	* **Method**: GET
+
+* **Delete a CustomDataKind**
+	* **URL**: http://{ip}:{port}/apis/v1/customdatakinds/{kind name}
+	* **Method**: DELETE
+
+* **Create a CustomData**
+	* **URL**: http://{ip}:{port}/apis/v1/customdata/{kind name}
+	* **Method**: POST
+	* **Body**: CustomData definition is YAML.
+
+* **Update a CustomData**
+	* **URL**: http://{ip}:{port}/apis/v1/customdata/{kind name}
+	* **Method**: PUT
+	* **Body**: CustomData definition is YAML.
+
+* **Query the definition of a CustomData**
+	* **URL**: http://{ip}:{port}/apis/v1/customdata/{kind name}/{data id}
+	* **Method**: GET
+
+* **List the definition of all CustomData of a kind**
+	* **URL**: http://{ip}:{port}/apis/v1/customdata/{kind name}
+	* **Method**: GET
+
+* **Delete a CustomData**
+	* **URL**: http://{ip}:{port}/apis/v1/customdata/{kind name}/{data id}
+	* **Method**: DELETE
+
+* **Delete all CustomData of a kind**
+	* **URL**: http://{ip}:{port}/apis/v1/customdata/{kind name}
+	* **Method**: DELETE
+
+* **Bulk update**
+	* **URL**: http://{ip}:{port}/apis/v1/customdata/{kind name}/items
+	* **Method**: POST
+	* **Body**: A change request in YAML, as defined below.
+
+```yaml
+rebuild: false
+delete: [data1, data2]
+list:
+- name: data3
+  field1: 12
+- name: data4
+  field1: foo
+```
+When `rebuild` is true (default is false), all existing data items are deleted before processing the data items in `list`. `delete` is an array of data identifiers to be deleted, this array is ignored when `rebuild` is true. `list` is an array of data items to be created or updated.