allow datasources to use other datasources

create a null data source for testing. We can choose to document if we want to, but it's a convenience for us

add a test to catch cyclic datasource dependency, update tests to include out of order data sources, and update the code to clean up the returned diagnostics generated from the recursive evaluation

PR review comments

command/plugin.go

@@ -17,6 +17,7 @@ import (
 	nullbuilder "github.com/hashicorp/packer/builder/null"
 	hcppackerimagedatasource "github.com/hashicorp/packer/datasource/hcp-packer-image"
 	hcppackeriterationdatasource "github.com/hashicorp/packer/datasource/hcp-packer-iteration"
+	nulldatasource "github.com/hashicorp/packer/datasource/null"
 	packerimageiterationdatasource "github.com/hashicorp/packer/datasource/packer-image-iteration"
 	artificepostprocessor "github.com/hashicorp/packer/post-processor/artifice"
 	checksumpostprocessor "github.com/hashicorp/packer/post-processor/checksum"
@@ -64,6 +65,7 @@ var PostProcessors = map[string]packersdk.PostProcessor{
 var Datasources = map[string]packersdk.Datasource{
 	"hcp-packer-image":       new(hcppackerimagedatasource.Datasource),
 	"hcp-packer-iteration":   new(hcppackeriterationdatasource.Datasource),
+	"null":                   new(nulldatasource.Datasource),
 	"packer-image-iteration": new(packerimageiterationdatasource.Datasource),
 }
 

datasource/hcp-packer-image/data.go

@@ -111,6 +111,7 @@ func (d *Datasource) Execute() (cty.Value, error) {
 	if err != nil {
 		return cty.NullVal(cty.EmptyObject), err
 	}
+
 	// Load channel.
 	log.Printf("[INFO] Reading info from HCP Packer registry (%s) [project_id=%s, organization_id=%s, iteration_id=%s]",
 		d.config.Bucket, cli.ProjectID, cli.OrganizationID, d.config.IterationID)
@@ -124,20 +125,21 @@ func (d *Datasource) Execute() (cty.Value, error) {
 	output := DatasourceOutput{}
 
 	for _, build := range iteration.Builds {
-		if build.CloudProvider == d.config.CloudProvider {
-			for _, image := range build.Images {
-				if image.Region == d.config.Region {
-					// This is the desired image.
-					output = DatasourceOutput{
-						CloudProvider: build.CloudProvider,
-						ComponentType: build.ComponentType,
-						CreatedAt:     image.CreatedAt.String(),
-						BuildID:       build.ID,
-						IterationID:   build.IterationID,
-						PackerRunUUID: build.PackerRunUUID,
-						ID:            image.ImageID,
-						Region:        image.Region,
-					}
+		if build.CloudProvider != d.config.CloudProvider {
+			continue
+		}
+		for _, image := range build.Images {
+			if image.Region == d.config.Region {
+				// This is the desired image.
+				output = DatasourceOutput{
+					CloudProvider: build.CloudProvider,
+					ComponentType: build.ComponentType,
+					CreatedAt:     image.CreatedAt.String(),
+					BuildID:       build.ID,
+					IterationID:   build.IterationID,
+					PackerRunUUID: build.PackerRunUUID,
+					ID:            image.ImageID,
+					Region:        image.Region,
 				}
 			}
 		}

datasource/null/data.go

@@ -0,0 +1,68 @@
+//go:generate packer-sdc struct-markdown
+//go:generate packer-sdc mapstructure-to-hcl2 -type DatasourceOutput,Config
+package null
+
+import (
+	"fmt"
+
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/hashicorp/hcl/v2/hcldec"
+	"github.com/hashicorp/packer-plugin-sdk/common"
+	"github.com/hashicorp/packer-plugin-sdk/hcl2helper"
+	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
+	"github.com/hashicorp/packer-plugin-sdk/template/config"
+)
+
+type Datasource struct {
+	config Config
+}
+
+// The Null data source is designed to demonstrate how data sources work, and
+// to provide a test plugin. It does not do anything useful; you assign an
+// input string and it gets returned as an output string.
+type Config struct {
+	common.PackerConfig `mapstructure:",squash"`
+	// This variable will get stored as "output" in the output spec.
+	Input string `mapstructure:"input" required:"true"`
+}
+
+func (d *Datasource) ConfigSpec() hcldec.ObjectSpec {
+	return d.config.FlatMapstructure().HCL2Spec()
+}
+
+func (d *Datasource) Configure(raws ...interface{}) error {
+	err := config.Decode(&d.config, nil, raws...)
+	if err != nil {
+		return err
+	}
+
+	var errs *packersdk.MultiError
+
+	if d.config.Input == "" {
+		errs = packersdk.MultiErrorAppend(errs, fmt.Errorf("The `input` must be specified"))
+	}
+
+	if errs != nil && len(errs.Errors) > 0 {
+		return errs
+	}
+	return nil
+}
+
+type DatasourceOutput struct {
+	// Output will return the input variable, as output.
+	Output string `mapstructure:"output"`
+}
+
+func (d *Datasource) OutputSpec() hcldec.ObjectSpec {
+	return (&DatasourceOutput{}).FlatMapstructure().HCL2Spec()
+}
+
+func (d *Datasource) Execute() (cty.Value, error) {
+	// Pass input variable through to output.
+	output := DatasourceOutput{
+		Output: d.config.Input,
+	}
+
+	return hcl2helper.HCL2ValueFromConfig(output, d.OutputSpec()), nil
+}

hcl2template/common_test.go

@@ -12,6 +12,7 @@ import (
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 	"github.com/hashicorp/packer-plugin-sdk/template/config"
 	"github.com/hashicorp/packer/builder/null"
+	dnull "github.com/hashicorp/packer/datasource/null"
 	. "github.com/hashicorp/packer/hcl2template/internal"
 	packerregistry "github.com/hashicorp/packer/internal/packer_registry"
 	"github.com/hashicorp/packer/packer"
@@ -41,6 +42,7 @@ func getBasicParser(opts ...getParserOption) *Parser {
 			},
 			DataSources: packer.MapOfDatasource{
 				"amazon-ami": func() (packersdk.Datasource, error) { return &MockDatasource{}, nil },
+				"null":       func() (packersdk.Datasource, error) { return &dnull.Datasource{}, nil },
 			},
 		},
 	}

hcl2template/testdata/datasources/dependency_cycle.pkr.hcl

@@ -0,0 +1,6 @@
+data "null" "gummy" {
+  input = "${data.null.bear.output}"
+}
+data "null" "bear" {
+  input = "${data.null.gummy.output}"
+}

hcl2template/testdata/datasources/recursive.pkr.hcl

@@ -0,0 +1,19 @@
+data "null" "foo" {
+  input = "chocolate"
+}
+
+data "null" "yummy" {
+  input = "${data.null.bang.output}-and-sprinkles"
+}
+
+data "null" "bar" {
+  input = "vanilla"
+}
+
+data "null" "baz" {
+  input = "${data.null.foo.output}-${data.null.bar.output}-swirl"
+}
+
+data "null" "bang" {
+  input = "${data.null.baz.output}-with-marshmallows"
+}

hcl2template/types.datasource.go

@@ -2,6 +2,7 @@ package hcl2template
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
@@ -62,7 +63,7 @@ func (ds *Datasources) Values() (map[string]cty.Value, hcl.Diagnostics) {
 	return res, diags
 }
 
-func (cfg *PackerConfig) startDatasource(dataSourceStore packer.DatasourceStore, ref DatasourceRef) (packersdk.Datasource, hcl.Diagnostics) {
+func (cfg *PackerConfig) startDatasource(dataSourceStore packer.DatasourceStore, ref DatasourceRef, secondaryEvaluation bool) (packersdk.Datasource, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
 	block := cfg.Datasources[ref].block
 
@@ -101,19 +102,48 @@ func (cfg *PackerConfig) startDatasource(dataSourceStore packer.DatasourceStore,
 			Severity: hcl.DiagError,
 		})
 	}
+
+	// HACK:
+	// This is where we parse the variables being used in the data sources.
+	// By passing in the DatasourceContext variable, we tell the EvalContext
+	// that since this is a datasource being evaluated, we should not allow
+	// other data sources to be decoded into it. When secondaryEvaluation is
+	// true, we know that this data source needs another data source in order
+	// to be evaluated. So we instead retrieve a different EvalContext.
+	// This is a brute force method to enable data sources to depend on each
+	// other, and a more elegant solution will be available once we implement a
+	// true DAG for Packer.
+	var decoded cty.Value
+	var moreDiags hcl.Diagnostics
 	body := block.Body
-	decoded, moreDiags := decodeHCL2Spec(body, cfg.EvalContext(DatasourceContext, nil), datasource)
+	if secondaryEvaluation {
+		// LocalContext is a lie! See above.
+		decoded, moreDiags = decodeHCL2Spec(body, cfg.EvalContext(LocalContext, nil), datasource)
+	} else {
+		decoded, moreDiags = decodeHCL2Spec(body, cfg.EvalContext(DatasourceContext, nil), datasource)
+	}
+
 	diags = append(diags, moreDiags...)
 	if moreDiags.HasErrors() {
-		return nil, diags
+		for _, err = range moreDiags.Errs() {
+			// If the error is just that there's no "data" object in the
+			// context, don't fail. We will track this data source for decoding
+			// again later, once we've evaluated all of the datasources.
+			// return nil, diags
+			if !strings.Contains(err.Error(), `There is no variable named "data"`) {
+				// There's an error that isn't just a recursive data source
+				// interpolation error
+				return nil, diags
+			}
+		}
 	}
 
-	// In case of cty.Unknown values, this will write a equivalent placeholder of the same type
-	// Unknown types are not recognized by the json marshal during the RPC call and we have to do this here
-	// to avoid json parsing failures when running the validate command.
-	// We don't do this before so we can validate if variable types matches correctly on decodeHCL2Spec.
+	// In case of cty.Unknown values, this will write a equivalent placeholder
+	// of the same type. Unknown types are not recognized by the json marshal
+	// during the RPC call and we have to do this here to avoid json parsing
+	// failures when running the validate command. We don't do this before so
+	// we can validate if variable type matches correctly on decodeHCL2Spec.
 	decoded = hcl2shim.WriteUnknownPlaceholderValues(decoded)
-
 	if err := datasource.Configure(decoded); err != nil {
 		diags = append(diags, &hcl.Diagnostic{
 			Summary:  err.Error(),