allow to discover & start packer-plugin-* muliplugin binaries (hashic…

…orp#10277)

This add :
* discovery of `packer-plugin-*` binaries from the known folders and ask them to describe themselves
* tests

For testing: in go we create a bash script that in turn calls back to Go. I could not make the tests to work on windows and then would like to postpone testing this for when we know more about the finite layout of this feature. That is mainly: how things are going to work with init, versioning and such.

config.go

@@ -141,107 +141,6 @@ func (c *config) StartProvisioner(name string) (packersdk.Provisioner, error) {
 	return c.Provisioners.Start(name)
 }
 
-func (c *config) discoverExternalComponents(path string) error {
-	var err error
-
-	if !filepath.IsAbs(path) {
-		path, err = filepath.Abs(path)
-		if err != nil {
-			return err
-		}
-	}
-	externallyUsed := []string{}
-
-	pluginPaths, err := c.discoverSingle(filepath.Join(path, "packer-builder-*"))
-	if err != nil {
-		return err
-	}
-	for pluginName, pluginPath := range pluginPaths {
-		newPath := pluginPath // this needs to be stored in a new variable for the func below
-		c.Builders[pluginName] = func() (packersdk.Builder, error) {
-			return c.Plugins.Client(newPath).Builder()
-		}
-		externallyUsed = append(externallyUsed, pluginName)
-	}
-	if len(externallyUsed) > 0 {
-		sort.Strings(externallyUsed)
-		log.Printf("using external builders %v", externallyUsed)
-		externallyUsed = nil
-	}
-
-	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-post-processor-*"))
-	if err != nil {
-		return err
-	}
-	for pluginName, pluginPath := range pluginPaths {
-		newPath := pluginPath // this needs to be stored in a new variable for the func below
-		c.PostProcessors[pluginName] = func() (packersdk.PostProcessor, error) {
-			return c.Plugins.Client(newPath).PostProcessor()
-		}
-		externallyUsed = append(externallyUsed, pluginName)
-	}
-	if len(externallyUsed) > 0 {
-		sort.Strings(externallyUsed)
-		log.Printf("using external post-processors %v", externallyUsed)
-		externallyUsed = nil
-	}
-
-	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-provisioner-*"))
-	if err != nil {
-		return err
-	}
-	for pluginName, pluginPath := range pluginPaths {
-		newPath := pluginPath // this needs to be stored in a new variable for the func below
-		c.Provisioners[pluginName] = func() (packersdk.Provisioner, error) {
-			return c.Plugins.Client(newPath).Provisioner()
-		}
-		externallyUsed = append(externallyUsed, pluginName)
-	}
-	if len(externallyUsed) > 0 {
-		sort.Strings(externallyUsed)
-		log.Printf("using external provisioners %v", externallyUsed)
-		externallyUsed = nil
-	}
-
-	return nil
-}
-
-func (c *config) discoverSingle(glob string) (map[string]string, error) {
-	matches, err := filepath.Glob(glob)
-	if err != nil {
-		return nil, err
-	}
-
-	res := make(map[string]string)
-
-	prefix := filepath.Base(glob)
-	prefix = prefix[:strings.Index(prefix, "*")]
-	for _, match := range matches {
-		file := filepath.Base(match)
-
-		// On Windows, ignore any plugins that don't end in .exe.
-		// We could do a full PATHEXT parse, but this is probably good enough.
-		if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" {
-			log.Printf(
-				"[DEBUG] Ignoring plugin match %s, no exe extension",
-				match)
-			continue
-		}
-
-		// If the filename has a ".", trim up to there
-		if idx := strings.Index(file, ".exe"); idx >= 0 {
-			file = file[:idx]
-		}
-
-		// Look for foo-bar-baz. The plugin name is "baz"
-		pluginName := file[len(prefix):]
-		log.Printf("[DEBUG] Discovered plugin: %s = %s", pluginName, match)
-		res[pluginName] = match
-	}
-
-	return res, nil
-}
-
 func (c *config) discoverInternalComponents() error {
 	// Get the packer binary path
 	packerPath, err := os.Executable()

packer-plugin-sdk/plugin/set.go

@@ -78,10 +78,10 @@ func (i *Set) RegisterProvisioner(name string, provisioner packersdk.Provisioner
 //  * "start post-processor example" starts the post-processor "example"
 func (i *Set) Run() error {
 	args := os.Args[1:]
-	return i.run(args...)
+	return i.RunCommand(args...)
 }
 
-func (i *Set) run(args ...string) error {
+func (i *Set) RunCommand(args ...string) error {
 	if len(args) < 1 {
 		return fmt.Errorf("needs at least one argument")
 	}

packer-plugin-sdk/plugin/set_test.go

@@ -50,7 +50,7 @@ func TestSet(t *testing.T) {
 		t.Fatalf("Unexpected description: %s", diff)
 	}
 
-	err := set.run("start", "builder", "example")
+	err := set.RunCommand("start", "builder", "example")
 	if diff := cmp.Diff(err.Error(), ErrManuallyStartedPlugin.Error()); diff != "" {
 		t.Fatalf("Unexpected error: %s", diff)
 	}

packer/plugin/discover.go

@@ -1,6 +1,7 @@
 package plugin
 
 import (
+	"encoding/json"
 	"log"
 	"os"
 	"os/exec"
@@ -159,6 +160,17 @@ func (c *Config) discoverExternalComponents(path string) error {
 		log.Printf("using external provisioners %v", externallyUsed)
 	}
 
+	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-plugin-*"))
+	if err != nil {
+		return err
+	}
+
+	for pluginName, pluginPath := range pluginPaths {
+		if err := c.discoverMultiPlugin(pluginName, pluginPath); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -175,6 +187,11 @@ func (c *Config) discoverSingle(glob string) (map[string]string, error) {
 	for _, match := range matches {
 		file := filepath.Base(match)
 
+		// skip folders like packer-plugin-sdk
+		if stat, err := os.Stat(file); err == nil && stat.IsDir() {
+			continue
+		}
+
 		// On Windows, ignore any plugins that don't end in .exe.
 		// We could do a full PATHEXT parse, but this is probably good enough.
 		if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" {
@@ -198,11 +215,62 @@ func (c *Config) discoverSingle(glob string) (map[string]string, error) {
 	return res, nil
 }
 
-func (c *Config) Client(path string) *Client {
+// discoverMultiPlugin takes the description from a multiplugin binary and
+// makes the plugins available to use in Packer. Each plugin found in the
+// binary will be addressable using `${pluginName}-${builderName}` for example.
+// pluginName could be manually set. It usually is a cloud name like amazon.
+// pluginName can be extrapolated from the filename of the binary; so
+// if the "packer-plugin-amazon" binary had an "ebs" builder one could use
+// the "amazon-ebs" builder.
+func (c *Config) discoverMultiPlugin(pluginName, pluginPath string) error {
+	out, err := exec.Command(pluginPath, "describe").Output()
+	if err != nil {
+		return err
+	}
+	var desc pluginsdk.SetDescription
+	if err := json.Unmarshal(out, &desc); err != nil {
+		return err
+	}
+
+	pluginPrefix := pluginName + "-"
+
+	for _, builderName := range desc.Builders {
+		builderName := builderName // copy to avoid pointer overwrite issue
+		c.builders[pluginPrefix+builderName] = func() (packersdk.Builder, error) {
+			return c.Client(pluginPath, "start", "builder", builderName).Builder()
+		}
+	}
+	if len(desc.Builders) > 0 {
+		log.Printf("found external %v builders from %s plugin", desc.Builders, pluginName)
+	}
+
+	for _, postProcessorName := range desc.PostProcessors {
+		postProcessorName := postProcessorName // copy to avoid pointer overwrite issue
+		c.postProcessors[pluginPrefix+postProcessorName] = func() (packersdk.PostProcessor, error) {
+			return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor()
+		}
+	}
+	if len(desc.PostProcessors) > 0 {
+		log.Printf("found external %v post-processors from %s plugin", desc.PostProcessors, pluginName)
+	}
+
+	for _, provisionerName := range desc.Provisioners {
+		provisionerName := provisionerName // copy to avoid pointer overwrite issue
+		c.provisioners[pluginPrefix+provisionerName] = func() (packersdk.Provisioner, error) {
+			return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner()
+		}
+	}
+	if len(desc.Provisioners) > 0 {
+		log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName)
+	}
+
+	return nil
+}
+
+func (c *Config) Client(path string, args ...string) *Client {
 	originalPath := path
 
 	// Check for special case using `packer plugin PLUGIN`
-	args := []string{}
 	if strings.Contains(path, PACKERSPACE) {
 		parts := strings.Split(path, PACKERSPACE)
 		path = parts[0]