Use default spec coorectly & fix httppipeline.Validate without recurs…

…ively checking (easegress-io#100)

* [validator]: Use default spec coorectly & fix httppipeline.Validate without recursively checking

go.mod

@@ -32,14 +32,16 @@ require (
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/rs/cors v1.7.0
+	github.com/sanity-io/litter v1.5.1 // indirect
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.8.0
 	github.com/tcnksm/go-httpstat v0.2.1-0.20191008022543-e866bb274419
 	github.com/tidwall/gjson v1.8.0
 	github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce
 	github.com/valyala/fasttemplate v1.2.1
-	github.com/xeipuuv/gojsonschema v1.2.0
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5
 	github.com/yl2chen/cidranger v1.0.2
 	go.etcd.io/etcd/api/v3 v3.5.0
 	go.etcd.io/etcd/client/v3 v3.5.0

go.sum

@@ -1139,6 +1139,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
+github.com/sanity-io/litter v1.5.1 h1:dwnrSypP6q56o3lFxTU+t2fwQ9A+U5qrXVO4Qg9KwVU=
+github.com/sanity-io/litter v1.5.1/go.mod h1:5Z71SvaYy5kcGtyglXOC9rrUi3c1E8CamFWjQsazTh0=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
@@ -1233,6 +1235,7 @@ github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -1300,11 +1303,15 @@ github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49
 github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5 h1:ImnGIsrcG8vwbovhYvvSY8fagVV6QhCWSWXfzwGDLVs=
+github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=

pkg/api/object.go

@@ -42,45 +42,42 @@ const (
 
 func (s *Server) objectAPIEntries() []*APIEntry {
 	return []*APIEntry{
-		&APIEntry{
+		{
 			Path:    ObjectKindsPrefix,
 			Method:  "GET",
 			Handler: s.listObjectKinds,
 		},
-
-		&APIEntry{
+		{
 			Path:    ObjectPrefix,
 			Method:  "POST",
 			Handler: s.createObject,
 		},
-		&APIEntry{
+		{
 			Path:    ObjectPrefix,
 			Method:  "GET",
 			Handler: s.listObjects,
 		},
-
-		&APIEntry{
+		{
 			Path:    ObjectPrefix + "/{name}",
 			Method:  "GET",
 			Handler: s.getObject,
 		},
-		&APIEntry{
+		{
 			Path:    ObjectPrefix + "/{name}",
 			Method:  "PUT",
 			Handler: s.updateObject,
 		},
-		&APIEntry{
+		{
 			Path:    ObjectPrefix + "/{name}",
 			Method:  "DELETE",
 			Handler: s.deleteObject,
 		},
-
-		&APIEntry{
+		{
 			Path:    StatusObjectPrefix,
 			Method:  "GET",
 			Handler: s.listStatusObjects,
 		},
-		&APIEntry{
+		{
 			Path:    StatusObjectPrefix + "/{name}",
 			Method:  "GET",
 			Handler: s.getStatusObject,

pkg/object/function/faascontroller.go

@@ -20,12 +20,10 @@ package function
 import (
 	"fmt"
 
-	"github.com/megaease/easegress/pkg/logger"
 	"github.com/megaease/easegress/pkg/object/function/spec"
 	"github.com/megaease/easegress/pkg/object/function/worker"
 	"github.com/megaease/easegress/pkg/supervisor"
 	"github.com/megaease/easegress/pkg/v"
-	"gopkg.in/yaml.v2"
 )
 
 const (
@@ -80,15 +78,7 @@ func (f *FaasController) Validate() error {
 		return fmt.Errorf("unknown faas provider: %s", f.spec.Provider)
 	}
 
-	buff, err := yaml.Marshal(f.spec.HTTPServer)
-	if err != nil {
-		err = fmt.Errorf("BUG: marshal %#v to yaml failed: %v",
-			f.spec.HTTPServer, err)
-		logger.Errorf(err.Error())
-		return err
-	}
-
-	vr := v.Validate(f.spec.HTTPServer, buff)
+	vr := v.Validate(f.spec.HTTPServer)
 	if !vr.Valid() {
 		return fmt.Errorf("%s", vr.Error())
 	}

pkg/object/function/worker/api.go

@@ -127,7 +127,7 @@ func (worker *Worker) readAPISpec(w http.ResponseWriter, r *http.Request, spec i
 		return fmt.Errorf("unmarshal failed: %v", err)
 	}
 
-	vr := v.Validate(spec, body)
+	vr := v.Validate(spec)
 	if !vr.Valid() {
 		return fmt.Errorf("validate failed: \n%s", vr.Error())
 	}

pkg/object/function/worker/ingress.go

@@ -242,8 +242,8 @@ func (ings *ingressServer) add(pipeline string) error {
 	index := ings.find(pipeline)
 	// not backend as function's pipeline
 	if index == -1 {
-		rule := httpserver.Rule{
-			Paths: []httpserver.Path{
+		rule := &httpserver.Rule{
+			Paths: []*httpserver.Path{
 				{
 					PathPrefix: "/",
 					Headers: []*httpserver.Header{

pkg/object/httppipeline/httppipeline.go

@@ -24,13 +24,12 @@ import (
 	"sync"
 	"time"
 
-	yaml "gopkg.in/yaml.v2"
-
 	"github.com/megaease/easegress/pkg/context"
 	"github.com/megaease/easegress/pkg/logger"
 	"github.com/megaease/easegress/pkg/protocol"
 	"github.com/megaease/easegress/pkg/supervisor"
 	"github.com/megaease/easegress/pkg/util/stringtool"
+	"github.com/megaease/easegress/pkg/util/yamltool"
 )
 
 const (
@@ -69,7 +68,7 @@ type (
 	// Spec describes the HTTPPipeline.
 	Spec struct {
 		Flow    []Flow                   `yaml:"flow" jsonschema:"omitempty"`
-		Filters []map[string]interface{} `yaml:"filters" jsonschema:"-"`
+		Filters []map[string]interface{} `yaml:"filters" jsonschema:"required"`
 	}
 
 	// Flow controls the flow of pipeline.
@@ -179,36 +178,21 @@ func deletePipelineContext(ctx context.HTTPContext) {
 	runningContexts.Delete(ctx)
 }
 
-func marshal(i interface{}) []byte {
-	buff, err := yaml.Marshal(i)
-	if err != nil {
-		panic(fmt.Errorf("marsharl %#v failed: %v", i, err))
-	}
-	return buff
-}
-
-func unmarshal(buff []byte, i interface{}) {
-	err := yaml.Unmarshal(buff, i)
-	if err != nil {
-		panic(fmt.Errorf("unmarshal failed: %v", err))
-	}
-}
-
 func extractFiltersData(config []byte) interface{} {
 	var whole map[string]interface{}
-	unmarshal(config, &whole)
+	yamltool.Unmarshal(config, &whole)
 	return whole["filters"]
 }
 
 func convertToFilterBuffs(obj interface{}) map[string][]byte {
 	var filters []map[string]interface{}
-	unmarshal(marshal(obj), &filters)
+	yamltool.Unmarshal(yamltool.Marshal(obj), &filters)
 
 	rst := make(map[string][]byte)
 	for _, p := range filters {
-		buff := marshal(p)
+		buff := yamltool.Marshal(p)
 		meta := &FilterMetaSpec{}
-		unmarshal(buff, meta)
+		yamltool.Unmarshal(buff, meta)
 		rst[meta.Name] = buff
 	}
 	return rst
@@ -230,17 +214,19 @@ func (meta *FilterMetaSpec) Validate() error {
 }
 
 // Validate validates Spec.
-func (s Spec) Validate(config []byte) (err error) {
+func (s Spec) Validate() (err error) {
 	errPrefix := "filters"
 	defer func() {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("%s: %s", errPrefix, r)
 		}
 	}()
 
+	config := yamltool.Marshal(s)
+
 	filtersData := extractFiltersData(config)
 	if filtersData == nil {
-		return fmt.Errorf("validate failed: filters is required")
+		return fmt.Errorf("filters is required")
 	}
 	filterBuffs := convertToFilterBuffs(filtersData)
 

pkg/object/httppipeline/spec.go

@@ -20,9 +20,8 @@ package httppipeline
 import (
 	"fmt"
 
-	yaml "gopkg.in/yaml.v2"
-
 	"github.com/megaease/easegress/pkg/supervisor"
+	"github.com/megaease/easegress/pkg/util/yamltool"
 	"github.com/megaease/easegress/pkg/v"
 )
 
@@ -46,43 +45,65 @@ type (
 )
 
 // NewFilterSpec creates a filter spec and validates it.
-func NewFilterSpec(rawSpec map[string]interface{}, super *supervisor.Supervisor) (*FilterSpec, error) {
-	s := &FilterSpec{
-		super:   super,
-		rawSpec: rawSpec,
-	}
+func NewFilterSpec(originalRawSpec map[string]interface{}, super *supervisor.Supervisor) (
+	s *FilterSpec, err error) {
 
-	yamlConfig, err := yaml.Marshal(rawSpec)
-	if err != nil {
-		return nil, fmt.Errorf("marshal %#v to yaml failed: %v", rawSpec, err)
-	}
+	s = &FilterSpec{super: super}
+
+	defer func() {
+		if r := recover(); r != nil {
+			s = nil
+			err = fmt.Errorf("%v", r)
+		} else {
+			err = nil
+		}
+	}()
 
-	s.yamlConfig = string(yamlConfig)
+	yamlBuff := yamltool.Marshal(originalRawSpec)
 
+	// Meta part.
 	meta := &FilterMetaSpec{}
-	err = yaml.Unmarshal(yamlConfig, meta)
-	if err != nil {
-		return nil, fmt.Errorf("unmarshal failed: %v", err)
+	yamltool.Unmarshal(yamlBuff, meta)
+	verr := v.Validate(meta)
+	if !verr.Valid() {
+		panic(verr)
 	}
 
+	// Filter self part.
 	rootFilter, exists := filterRegistry[meta.Kind]
 	if !exists {
-		return nil, fmt.Errorf("kind %s not found", meta.Kind)
+		panic(fmt.Errorf("kind %s not found", meta.Kind))
+	}
+	filterSpec := rootFilter.DefaultSpec()
+	yamltool.Unmarshal(yamlBuff, filterSpec)
+	verr = v.Validate(filterSpec)
+	if !verr.Valid() {
+		// TODO: Make the invalid part more accurate. e,g:
+		// filters: jsonschemaErrs:
+		// - 'policies.0: name is required'
+		// to
+		// filters: jsonschemaErrs:
+		// - 'rateLimiter.policies.0: name is required'
+		panic(verr)
 	}
 
-	s.meta, s.filterSpec, s.rootFilter = meta, rootFilter.DefaultSpec(), rootFilter
+	// Build final yaml config and raw spec.
+	var rawSpec map[string]interface{}
+	filterBuff := yamltool.Marshal(filterSpec)
+	yamltool.Unmarshal(filterBuff, &rawSpec)
 
-	err = yaml.Unmarshal(yamlConfig, s.filterSpec)
-	if err != nil {
-		return nil, fmt.Errorf("unmarshal failed: %v", err)
-	}
+	metaBuff := yamltool.Marshal(meta)
+	yamltool.Unmarshal(metaBuff, &rawSpec)
 
-	vr := v.Validate(s.filterSpec, []byte(yamlConfig))
-	if !vr.Valid() {
-		return nil, fmt.Errorf("%v", vr.Error())
-	}
+	yamlConfig := string(yamltool.Marshal(rawSpec))
+
+	s.meta = meta
+	s.filterSpec = filterSpec
+	s.rawSpec = rawSpec
+	s.yamlConfig = yamlConfig
+	s.rootFilter = rootFilter
 
-	return s, nil
+	return
 }
 
 func (s *FilterSpec) Super() *supervisor.Supervisor {

pkg/object/httpserver/mux.go

@@ -335,11 +335,11 @@ func (m *mux) reloadRules(superSpec *supervisor.Spec, muxMapper protocol.MuxMapp
 
 		paths := make([]*muxPath, len(specRule.Paths))
 		for j := 0; j < len(paths); j++ {
-			paths[j] = newMuxPath(ruleIPFilterChain, &specRule.Paths[j])
+			paths[j] = newMuxPath(ruleIPFilterChain, specRule.Paths[j])
 		}
 
 		// NOTE: Given the parent ipFilters not its own.
-		rules.rules[i] = newMuxRule(rules.ipFilterChan, &specRule, paths)
+		rules.rules[i] = newMuxRule(rules.ipFilterChan, specRule, paths)
 	}
 
 	m.rules.Store(rules)

pkg/object/httpserver/spec.go

@@ -51,7 +51,7 @@ type (
 		Keys map[string]string `yaml:"keys" jsonschema:"omitempty"`
 
 		IPFilter *ipfilter.Spec `yaml:"ipFilter,omitempty" jsonschema:"omitempty"`
-		Rules    []Rule         `yaml:"rules" jsonschema:"omitempty"`
+		Rules    []*Rule        `yaml:"rules" jsonschema:"omitempty"`
 	}
 
 	// Rule is first level entry of router.
@@ -66,7 +66,7 @@ type (
 		IPFilter   *ipfilter.Spec `yaml:"ipFilter,omitempty" jsonschema:"omitempty"`
 		Host       string         `yaml:"host" jsonschema:"omitempty"`
 		HostRegexp string         `yaml:"hostRegexp" jsonschema:"omitempty,format=regexp"`
-		Paths      []Path         `yaml:"paths" jsonschema:"omitempty"`
+		Paths      []*Path        `yaml:"paths" jsonschema:"omitempty"`
 	}
 
 	// Path is second level entry of router.