Openapi v3 (goadesign#2607)

* OpenAPI v3 initial commit

* Build Server for OpenAPI v3

* get host variable name and support retrieving multiple variables

* Move OpenAPI V3 data structures to Goa

* Use MatchString for matching string and we no longer return error for openapiv3 init

* Started Components and body types work

* Add security requirements and custom YAML marshaller/unmarshaller for Ref

* Add paths and operations

* Move URI scheme and parameter substitution to expressions

* Remove regex to find URI scheme

* Initial implementation of body types

* Add back buildExternalDocs (whoops)

* Finalize hashing algo

* move v2 openapi builder to v2 folder

* Update JSON Schema version

* Rework a bit openapi package structure

* Use openapi Schema in v3

* Continue OpenAPI v3 paths work

* Update dependencies

* clean up

* More progress on OpenAPI v3 support

* Add support for file servers

* Move openapi v2 tests to its own package

f

* Start adding openapiv3 tests

* more openapiv3 swagger gen tests

* Save work

* Add more tests and fix issues.

* More OpenAPI v3 testing and fixes

* Wrap up initial OpenAPI v3 support

* Fix broken tests

Co-authored-by: Nitin Mohan <[email protected]>
Co-authored-by: NeenuAVarghese <[email protected]>
Co-authored-by: Raphael Simon <[email protected]>

Author: 4 people

.golint_exclude

@@ -1,9 +1,11 @@
 dsl[/\]http.go
 expr[/\]http_response.go
 codegen[/\]service[/\]testing[/\].*
+http[/\]codegen[/\]codegen[/\].openapi[/\]v3[/\]ref.go
 http[/\]codegen[/\]testing[/\].*
 http[/\]middleware[/\]trace.go
 http[/\]middleware[/\]xray[/\]middleware.go
 http[/\]middleware[/\]xray[/\]wrap_doer.go
 http[/\]middleware[/\]xray[/\]wrap_doer_test.go
 grpc[/\]pb[/\].*
+

Makefile

@@ -28,7 +28,8 @@ DEPEND=\
 	github.com/golang/protobuf/protoc-gen-go \
 	github.com/golang/protobuf/proto \
 	honnef.co/go/tools/cmd/staticcheck \
-	github.com/hashicorp/go-getter/cmd/go-getter
+	github.com/hashicorp/go-getter/cmd/go-getter \
+	github.com/getkin/kin-openapi
 
 all: lint test
 

codegen/service/example_svc.go

@@ -115,10 +115,8 @@ func basicEndpointSection(m *expr.MethodExpr, svcData *Data) *codegen.SectionTem
 		ed.ResultIsStruct = expr.IsObject(m.Result.Type)
 		if md.ViewedResult != nil {
 			view := "default"
-			if m.Result.Meta != nil {
-				if v, ok := m.Result.Meta["view"]; ok {
-					view = v[0]
-				}
+			if v, ok := m.Result.Meta["view"]; ok {
+				view = v[0]
 			}
 			ed.ResultView = view
 		}

dsl/error.go

@@ -76,10 +76,7 @@ func Temporary() {
 		eval.IncompatibleDSL()
 		return
 	}
-	if attr.Meta == nil {
-		attr.Meta = make(expr.MetaExpr)
-	}
-	attr.Meta["goa:error:temporary"] = nil
+	attr.AddMeta("goa:error:temporary")
 }
 
 // Timeout qualifies an error type as describing errors due to timeouts.
@@ -101,10 +98,7 @@ func Timeout() {
 		eval.IncompatibleDSL()
 		return
 	}
-	if attr.Meta == nil {
-		attr.Meta = make(expr.MetaExpr)
-	}
-	attr.Meta["goa:error:timeout"] = nil
+	attr.AddMeta("goa:error:timeout")
 }
 
 // Fault qualifies an error type as describing errors due to a server-side
@@ -127,8 +121,5 @@ func Fault() {
 		eval.IncompatibleDSL()
 		return
 	}
-	if attr.Meta == nil {
-		attr.Meta = make(expr.MetaExpr)
-	}
-	attr.Meta["goa:error:fault"] = nil
+	attr.AddMeta("goa:error:fault")
 }

dsl/http.go

@@ -934,11 +934,7 @@ func Body(args ...interface{}) {
 			return
 		}
 		attr = expr.DupAtt(attr)
-		if attr.Meta == nil {
-			attr.Meta = expr.MetaExpr{"origin:attribute": []string{a}}
-		} else {
-			attr.Meta["origin:attribute"] = []string{a}
-		}
+		attr.AddMeta("origin:attribute", a)
 		if rt, ok := attr.Type.(*expr.ResultTypeExpr); ok {
 			// If the attribute type is a result type add the type to the
 			// GeneratedTypes so that the type's DSLFunc is executed.
@@ -976,10 +972,7 @@ func Body(args ...interface{}) {
 	if fn != nil {
 		eval.Execute(fn, attr)
 	}
-	if attr.Meta == nil {
-		attr.Meta = expr.MetaExpr{}
-	}
-	attr.Meta["http:body"] = []string{}
+	attr.AddMeta("http:body")
 	setter(attr)
 }
 
@@ -1177,8 +1170,5 @@ func params(exp eval.Expression) *expr.MappedAttributeExpr {
 // a HTTP cookie attribute for use by the HTTP code generator.
 func cookieAttribute(name, value string) {
 	c := eval.Current().(*expr.HTTPResponseExpr).Cookies
-	if c.Meta == nil {
-		c.Meta = expr.MetaExpr{}
-	}
-	c.Meta["cookie:"+name] = []string{value}
+	c.AddMeta("cookie:"+name, value)
 }

dsl/result_type.go

@@ -104,10 +104,7 @@ func TypeName(name string) {
 	case expr.UserType:
 		e.Rename(name)
 	case *expr.AttributeExpr:
-		if e.Meta == nil {
-			e.Meta = make(expr.MetaExpr)
-		}
-		e.Meta["struct:type:name"] = []string{name}
+		e.AddMeta("struct:type:name", name)
 	default:
 		eval.IncompatibleDSL()
 	}
@@ -209,10 +206,7 @@ func View(name string, adsl ...func()) {
 		}
 
 	case *expr.AttributeExpr:
-		if e.Meta == nil {
-			e.Meta = make(map[string][]string)
-		}
-		e.Meta["view"] = []string{name}
+		e.AddMeta("view", name)
 
 	default:
 		eval.IncompatibleDSL()
@@ -485,12 +479,7 @@ func buildView(name string, mt *expr.ResultTypeExpr, at *expr.AttributeExpr) (*e
 		cat := nat.Attribute
 		if existing := mt.Find(n); existing != nil {
 			dup := expr.DupAtt(existing)
-			if dup.Meta == nil {
-				dup.Meta = make(map[string][]string)
-			}
-			if len(cat.Meta["view"]) > 0 {
-				dup.Meta["view"] = cat.Meta["view"]
-			}
+			dup.AddMeta("view", cat.Meta["view"]...)
 			o.Set(n, dup)
 		} else if n != "links" {
 			return nil, fmt.Errorf("unknown attribute %#v", n)

expr/attribute.go

@@ -171,13 +171,6 @@ func TaggedAttribute(a *AttributeExpr, tag string) string {
 	return ""
 }
 
-// Prepare initializes the Meta expression.
-func (a *AttributeExpr) Prepare() {
-	if a.Meta == nil {
-		a.Meta = MetaExpr{}
-	}
-}
-
 // Validate tests whether the attribute required fields exist.  Since attributes
 // are unaware of their context, additional context information can be provided
 // to be used in error messages.  The parent definition context is automatically
@@ -504,6 +497,14 @@ func (a *AttributeExpr) Delete(name string) {
 	}
 }
 
+// AddMeta adds values to the meta field of the attribute.
+func (a *AttributeExpr) AddMeta(name string, vals ...string) {
+	if a.Meta == nil {
+		a.Meta = make(MetaExpr)
+	}
+	a.Meta[name] = append(a.Meta[name], vals...)
+}
+
 // Debug dumps the attribute to STDOUT in a goa developer friendly way.
 func (a *AttributeExpr) Debug(prefix string) { a.debug(prefix, make(map[*AttributeExpr]int), 0) }
 func (a *AttributeExpr) debug(prefix string, seen map[*AttributeExpr]int, indent int) {

expr/http_body_types.go

@@ -91,7 +91,7 @@ func httpRequestBody(a *HTTPEndpointExpr) *AttributeExpr {
 		return &AttributeExpr{Type: Empty}
 	}
 
-	// 2. Remove header and param attributes
+	// 2. Remove header, param and cookies attributes
 	body := NewMappedAttributeExpr(payload)
 	removeAttributes(body, headers)
 	removeAttributes(body, cookies)
@@ -225,6 +225,13 @@ func buildHTTPResponseBody(name string, attr *AttributeExpr, resp *HTTPResponseE
 		TypeName:      name,
 		UID:           concat(svc.Name(), "#", name),
 	}
+
+	// Remember original type name for example to generate friendly OpenAPI
+	// specs.
+	if t, ok := attr.Type.(UserType); ok {
+		userType.AttributeExpr.AddMeta("name:original", t.Name())
+	}
+
 	appendSuffix(userType.Attribute().Type, suffix)
 	rt, isrt := attr.Type.(*ResultTypeExpr)
 	if !isrt {

expr/http_endpoint.go

@@ -258,10 +258,10 @@ func (e *HTTPEndpointExpr) Prepare() {
 		}
 	}
 
-	// Make sure there's a default response if none define explicitly
+	// Make sure there's a default success response if none define explicitly.
 	if len(e.Responses) == 0 {
 		status := StatusOK
-		if e.MethodExpr.Payload.Type == Empty && !e.SkipResponseBodyEncodeDecode {
+		if e.MethodExpr.Result.Type == Empty && !e.SkipResponseBodyEncodeDecode {
 			status = StatusNoContent
 		}
 		e.Responses = []*HTTPResponseExpr{{StatusCode: status}}

expr/http_response.go

@@ -281,6 +281,10 @@ func (r *HTTPResponseExpr) Finalize(a *HTTPEndpointExpr, svcAtt *AttributeExpr)
 					r.Body.Validation.AddRequired(n)
 				}
 			}
+			// Remember original name for example to generate friendlier OpenAPI specs.
+			if t, ok := r.Body.Type.(UserType); ok {
+				t.Attribute().AddMeta("name:original", t.Name())
+			}
 			// Wrap object with user type to simplify response rendering code.
 			r.Body.Type = &UserTypeExpr{
 				AttributeExpr: DupAtt(r.Body),

expr/result_type.go

@@ -202,6 +202,11 @@ func (m *ResultTypeExpr) Finalize() {
 // The resulting result type defines a default view. The result type identifier is
 // computed by adding a parameter called "view" to the original identifier. The
 // value of the "view" parameter is the name of the view.
+//
+// Project returns an error if the view does not exist for the given result type
+// or any result type that makes up its attributes recursively. Note that
+// individual attributes may use a different view. In this case Project uses
+// that view and returns an error if it isn't defined on the attribute type.
 func Project(m *ResultTypeExpr, view string, seen ...map[string]*AttributeExpr) (*ResultTypeExpr, error) {
 	_, params, _ := mime.ParseMediaType(m.Identifier)
 	if params["view"] == view {

expr/server.go

@@ -126,7 +126,7 @@ var validSchemes = map[string]struct{}{"http": {}, "https": {}, "grpc": {}, "grp
 func (h *HostExpr) Validate() error {
 	verr := new(eval.ValidationErrors)
 	if len(h.URIs) == 0 {
-		verr.Add(h, "host must defined at least one URI")
+		verr.Add(h, "host must define at least one URI")
 	}
 	for _, u := range h.URIs {
 		vu := uriVariableRegex.ReplaceAllString(string(u), "/w")
@@ -184,21 +184,7 @@ func (h *HostExpr) Attribute() *AttributeExpr {
 func (h *HostExpr) Schemes() []string {
 	schemes := make(map[string]struct{})
 	for _, uri := range h.URIs {
-		ustr := string(uri)
-		// Did not use url package to find scheme because the url may
-		// contain params (i.e. http://{version}.example.com) which needs
-		// substition for url.Parse to succeed. Also URIs in host must have
-		// a scheme otherwise validations would have failed.
-		switch {
-		case strings.HasPrefix(ustr, "https"):
-			schemes["https"] = struct{}{}
-		case strings.HasPrefix(ustr, "http"):
-			schemes["http"] = struct{}{}
-		case strings.HasPrefix(ustr, "grpcs"):
-			schemes["grpcs"] = struct{}{}
-		case strings.HasPrefix(ustr, "grpc"):
-			schemes["grpc"] = struct{}{}
-		}
+		schemes[uri.Scheme()] = struct{}{}
 	}
 	ss := make([]string, len(schemes))
 	i := 0
@@ -236,6 +222,38 @@ func (h *HostExpr) HasGRPCScheme() bool {
 	return false
 }
 
+// URIString returns a valid URI string by substituting the parameters with
+// their default value if present or the first item in their enum. It returns
+// an error if the given URI expression is not found in the host URIs.
+func (h *HostExpr) URIString(u URIExpr) (string, error) {
+	found := false
+	for _, ue := range h.URIs {
+		if ue == u {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return "", fmt.Errorf("uri %s not found in host", string(u))
+	}
+	uri := string(u)
+	// Substitute URI parameters with the corresponding variables defined in
+	// the host expression. Validations would have made sure that every
+	// URI parameter have a corresponding variable.
+	for _, p := range u.Params() {
+		for _, v := range *AsObject(h.Variables.Type) {
+			if p == v.Name {
+				def := v.Attribute.DefaultValue
+				if def == nil {
+					def = v.Attribute.Validation.Values[0]
+				}
+				uri = strings.Replace(uri, fmt.Sprintf("{%s}", p), fmt.Sprintf("%v", def), -1)
+			}
+		}
+	}
+	return uri, nil
+}
+
 // Params return the names of the parameters used in URI if any.
 func (u URIExpr) Params() []string {
 	r := regexp.MustCompile(`\{([^\{\}]+)\}`)
@@ -249,3 +267,25 @@ func (u URIExpr) Params() []string {
 	}
 	return wcs
 }
+
+// Scheme returns the URI scheme. Possible values are http, https, grpc, and
+// grpcs.
+func (u URIExpr) Scheme() string {
+	ustr := string(u)
+	// Did not use url package to find scheme because the url may
+	// contain params (i.e. http://{version}.example.com) which needs
+	// substition for url.Parse to succeed. Also URIs in host must have
+	// a scheme otherwise validations would have failed.
+	switch {
+	case strings.HasPrefix(ustr, "https"):
+		return "https"
+	case strings.HasPrefix(ustr, "grpcs"):
+		return "grpcs"
+	case strings.HasPrefix(ustr, "grpc"):
+		return "grpc"
+	default:
+		// No need to worry about other values because the URIExpr would have failed
+		// validation.
+		return "http"
+	}
+}

expr/server_test.go

@@ -34,7 +34,7 @@ func TestServerExprValidate(t *testing.T) {
 		validURIs = []URIExpr{
 			validURI,
 		}
-		errNoURI            = fmt.Errorf("host must defined at least one URI")
+		errNoURI            = fmt.Errorf("host must define at least one URI")
 		errServiceUndefined = fmt.Errorf("service %q undefined", bar)
 	)
 
@@ -173,7 +173,7 @@ func TestHostExprValidate(t *testing.T) {
 				Type: d,
 			}
 		}
-		errNoURI                          = fmt.Errorf("host must defined at least one URI")
+		errNoURI                          = fmt.Errorf("host must define at least one URI")
 		errMalformedURI                   = fmt.Errorf("malformed URI %q", malformedURI)
 		errMissingSchemeURI               = fmt.Errorf("missing scheme for URI %q, scheme must be one of 'http', 'https', 'grpc' or 'grpcs'", missingSchemeURI)
 		errInvalidSchemeURI               = fmt.Errorf("invalid scheme for URI %q, scheme must be one of 'http', 'https', 'grpc' or 'grpcs'", invalidSchemeURI)

expr/service.go

@@ -137,11 +137,7 @@ func (e *ErrorExpr) Finalize() {
 			// This type does not have an attribute with "struct:error:name" meta.
 			// It means the type is used by at most one error (otherwise validations
 			// would have failed).
-			datt := dt.Attribute()
-			if datt.Meta == nil {
-				datt.Meta = MetaExpr{}
-			}
-			datt.Meta["struct:error:name"] = []string{e.Name}
+			dt.Attribute().AddMeta("struct:error:name", e.Name)
 		}
 	default:
 		ut := &UserTypeExpr{

expr/user_type.go

@@ -40,7 +40,11 @@ func (u *UserTypeExpr) Name() string {
 }
 
 // Rename changes the type name to the given value.
-func (u *UserTypeExpr) Rename(n string) { u.TypeName = n }
+func (u *UserTypeExpr) Rename(n string) {
+	// Remember original name for example to generate friendly docs.
+	u.AttributeExpr.AddMeta("name:original", u.TypeName)
+	u.TypeName = n
+}
 
 // IsCompatible returns true if u describes the (Go) type of val.
 func (u *UserTypeExpr) IsCompatible(val interface{}) bool {

go.mod

@@ -5,18 +5,23 @@ go 1.14
 require (
 	github.com/dimfeld/httppath v0.0.0-20170720192232-ee938bf73598
 	github.com/dimfeld/httptreemux/v5 v5.0.2
+	github.com/getkin/kin-openapi v0.15.0
 	github.com/go-openapi/loads v0.19.5
 	github.com/golang/protobuf v1.4.2
 	github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 // indirect
 	github.com/gorilla/websocket v1.4.1
+	github.com/hashicorp/go-getter v1.4.1 // indirect
 	github.com/manveru/faker v0.0.0-20171103152722-9fbc68a78c4d
 	github.com/manveru/gobdd v0.0.0-20131210092515-f1a17fdd710b // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/sergi/go-diff v1.1.0
 	github.com/smartystreets/goconvey v1.6.4 // indirect
+	github.com/stretchr/testify v1.5.1 // indirect
 	github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea
+	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 // indirect
-	golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f
+	golang.org/x/tools v0.0.0-20200619210111-0f592d2728bb
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/yaml.v2 v2.3.0
+	honnef.co/go/tools v0.0.1-2020.1.4 // indirect
 )