make toml an optional ytt module

so that ytt integrators do not have to take a dependency on toml

examples/integrating-with-ytt/internal-templating/go.mod

@@ -2,12 +2,12 @@ module example_internal_templating
 
 go 1.17
 
+// ensure example works with this copy of ytt; remove before use
 replace github.com/vmware-tanzu/carvel-ytt => ../../../
 
 require github.com/vmware-tanzu/carvel-ytt v0.40.1
 
 require (
-	github.com/BurntSushi/toml v1.0.0 // indirect
 	github.com/hashicorp/go-version v1.4.0 // indirect
 	github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368 // indirect
 )

examples/integrating-with-ytt/internal-templating/main.go

@@ -82,4 +82,3 @@ func templatesAsInput(tpl... string) (yttcmd.Input, error) {
 
 type noopWriter struct{}
 func (w noopWriter) Write(data []byte) (int, error) { return len(data), nil }
-

examples/toml-serialize/config.yml

@@ -0,0 +1,12 @@
+#@ load("@ytt:toml", "toml")
+
+#@ def data():
+map:
+  foo: bar
+array:
+- 1
+- 2
+- "three"
+#@ end
+
+toml: #@ toml.encode(data())

examples/toml-serialize/expected.txt

@@ -0,0 +1,5 @@
+toml: |
+  array = [1, 2, "three"]
+
+  [map]
+    foo = "bar"

pkg/cmd/template.go

@@ -6,6 +6,9 @@ package cmd
 import (
 	"github.com/spf13/cobra"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/cmd/template"
+
+	// Load ytt library extensions (should be available in ytt binary)
+	_ "github.com/vmware-tanzu/carvel-ytt/pkg/yttlibraryext"
 )
 
 // NewCmd construct main ytt command. It has been moved out of "template" package

pkg/cmd/template/cmd.go

@@ -54,7 +54,7 @@ func NewOptions() *Options {
 	return &Options{}
 }
 
-// BindFlags registers top level template flags for template command.
+// BindFlags registers template flags for template command.
 func (o *Options) BindFlags(cmdFlags CmdFlags) {
 	cmdFlags.BoolVar(&o.IgnoreUnknownComments, "ignore-unknown-comments", false,
 		"Configure whether unknown comments are considered as errors (comments that do not start with '#@' or '#!')")

pkg/yamltemplate/template_test.go

@@ -20,6 +20,8 @@ import (
 	"github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/yamltemplate"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/yttlibrary"
+
+	_ "github.com/vmware-tanzu/carvel-ytt/pkg/yttlibraryext"
 )
 
 var (

pkg/yttlibrary/all.go

@@ -7,18 +7,38 @@ import (
 	"fmt"
 
 	"github.com/k14s/starlark-go/starlark"
+	"github.com/k14s/starlark-go/starlarkstruct"
 	tplcore "github.com/vmware-tanzu/carvel-ytt/pkg/template/core"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/yttlibrary/overlay"
 )
 
+var registeredExts []*starlarkstruct.Module
+
+// RegisterExt adds "mod" to the standard set of ytt library modules as an extension.
+// An "extension" is a Starlark module that has external Go dependencies
+// (as opposed to other Starlark modules in the ytt library that either
+// have no dependencies or depend on the Go standard lib).
+// This enables those using ytt as a Go module to opt-in (rather than be forced)
+// to accept such dependencies. Only Carvel-maintained extensions can be registered;
+// this reserves the `@ytt:` namespace. Integrators who want to write their own extensions
+// should construct their own library.
+func RegisterExt(mod *starlarkstruct.Module) {
+	switch mod.Name {
+	case "toml":
+		registeredExts = append(registeredExts, mod)
+	default:
+		panic("ytt library namespace can only be extended with ytt modules")
+	}
+}
+
 type API struct {
 	modules map[string]starlark.StringDict
 }
 
 func NewAPI(replaceNodeFunc tplcore.StarlarkFunc, dataMod DataModule,
 	libraryMod starlark.StringDict) API {
 
-	return API{map[string]starlark.StringDict{
+	std := map[string]starlark.StringDict{
 		"assert": AssertAPI,
 		"regexp": RegexpAPI,
 
@@ -29,7 +49,6 @@ func NewAPI(replaceNodeFunc tplcore.StarlarkFunc, dataMod DataModule,
 		// Serializations
 		"base64": Base64API,
 		"json":   JSONAPI,
-		"toml":   TOMLAPI,
 		"yaml":   YAMLAPI,
 		"url":    URLAPI,
 		"ip":     IPAPI,
@@ -47,7 +66,17 @@ func NewAPI(replaceNodeFunc tplcore.StarlarkFunc, dataMod DataModule,
 		"version": VersionAPI,
 
 		"library": libraryMod,
-	}}
+	}
+
+	for _, ext := range registeredExts {
+		// Double check that we are not overriding predefined library
+		if _, found := std[ext.Name]; found {
+			panic("Internal inconsistency: shadowing ytt library with an extension module")
+		}
+		std[ext.Name] = starlark.StringDict{ext.Name: ext}
+	}
+
+	return API{std}
 }
 
 func (a API) FindModule(module string) (starlark.StringDict, error) {

pkg/yttlibraryext/all.go

@@ -0,0 +1,14 @@
+// Copyright 2022 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// ytt library extensions are defined in this package.
+// They have been separated from "yttlibrary" package because
+// they depend on functionality outside of Go standard library.
+// Users who import ytt as a library in their Go programs may not
+// want to depend on such functionality.
+
+package yttlibraryext
+
+import (
+	_ "github.com/vmware-tanzu/carvel-ytt/pkg/yttlibraryext/toml" // include toml
+)

pkg/yttlibrary/toml.go → pkg/yttlibraryext/toml/toml.go

@@ -1,7 +1,7 @@
 // Copyright 2021 VMware, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-package yttlibrary
+package toml
 
 import (
 	"bytes"
@@ -14,21 +14,26 @@ import (
 	"github.com/vmware-tanzu/carvel-ytt/pkg/orderedmap"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/template/core"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta"
+	"github.com/vmware-tanzu/carvel-ytt/pkg/yttlibrary"
 )
 
 var (
-	// TOMLAPI contains the definition of the @ytt:toml module
-	TOMLAPI = starlark.StringDict{
-		"toml": &starlarkstruct.Module{
-			Name: "toml",
-			Members: starlark.StringDict{
-				"encode": starlark.NewBuiltin("toml.encode", core.ErrWrapper(tomlModule{}.Encode)),
-				"decode": starlark.NewBuiltin("toml.decode", core.ErrWrapper(tomlModule{}.Decode)),
-			},
+	tomlMod = &starlarkstruct.Module{
+		Name: "toml",
+		Members: starlark.StringDict{
+			"encode": starlark.NewBuiltin("toml.encode", core.ErrWrapper(tomlModule{}.Encode)),
+			"decode": starlark.NewBuiltin("toml.decode", core.ErrWrapper(tomlModule{}.Decode)),
 		},
 	}
+
+	// TOMLAPI contains the definition of the @ytt:toml module
+	TOMLAPI = starlark.StringDict{"toml": tomlMod}
 )
 
+func init() {
+	yttlibrary.RegisterExt(tomlMod)
+}
+
 type tomlModule struct{}
 
 // Encode is a core.StarlarkFunc that renders the provided input into a TOML formatted string

test/e2e/e2e_test.go

@@ -133,6 +133,9 @@ func TestDifferentUsages(t *testing.T) {
 		"k8s-overlay-all-containers",
 		"k8s-overlay-remove-resources",
 		"k8s-overlay-in-config-map",
+		// test that @ytt:toml module works in default ytt binary
+		// as it is loaded in a different way than other @ytt:* modules.
+		"toml-serialize",
 	}
 	for _, k8sDirToTest := range dirs {
 		t.Run(fmt.Sprintf("k8s: %s", k8sDirToTest), func(t *testing.T) {