bake: allow variables across files

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

bake/bake.go

@@ -59,14 +59,11 @@ func ReadLocalFiles(names []string) ([]File, error) {
 }
 
 func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) {
-	var c Config
-	for _, f := range files {
-		cfg, err := ParseFile(f.Data, f.Name)
-		if err != nil {
-			return nil, err
-		}
-		c = mergeConfig(c, *cfg)
+	c, err := parseFiles(files)
+	if err != nil {
+		return nil, err
 	}
+
 	o, err := c.newOverrides(overrides)
 	if err != nil {
 		return nil, err
@@ -86,25 +83,73 @@ func ReadTargets(ctx context.Context, files []File, targets, overrides []string)
 	return m, nil
 }
 
+func parseFiles(files []File) (*Config, error) {
+	var c Config
+	var fs []*hcl.File
+	var scs []*StaticConfig
+	for _, f := range files {
+		cfg, f, sc, err := parseFile(f.Data, f.Name)
+		if err != nil {
+			return nil, err
+		}
+		if cfg != nil {
+			c = mergeConfig(c, *cfg)
+		} else {
+			fs = append(fs, f)
+			scs = append(scs, sc)
+		}
+	}
+
+	if len(fs) > 0 {
+		cfg, err := ParseHCL(hcl.MergeFiles(fs), mergeStaticConfig(scs))
+		if err != nil {
+			return nil, err
+		}
+		c = mergeConfig(c, dedupeConfig(*cfg))
+	}
+	return &c, nil
+}
+
+func dedupeConfig(c Config) Config {
+	c2 := c
+	c2.Targets = make([]*Target, 0, len(c2.Targets))
+	m := map[string]*Target{}
+	for _, t := range c.Targets {
+		if t2, ok := m[t.Name]; ok {
+			merge(t2, t)
+		} else {
+			m[t.Name] = t
+			c2.Targets = append(c2.Targets, t)
+		}
+	}
+	return c2
+}
+
 func ParseFile(dt []byte, fn string) (*Config, error) {
+	return parseFiles([]File{{Data: dt, Name: fn}})
+}
+
+func parseFile(dt []byte, fn string) (*Config, *hcl.File, *StaticConfig, error) {
 	fnl := strings.ToLower(fn)
 	if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
-		return ParseCompose(dt)
+		c, err := ParseCompose(dt)
+		return c, nil, nil, err
 	}
 
 	if strings.HasSuffix(fnl, ".json") || strings.HasSuffix(fnl, ".hcl") {
-		return ParseHCL(dt, fn)
+		f, sc, err := ParseHCLFile(dt, fn)
+		return nil, f, sc, err
 	}
 
 	cfg, err := ParseCompose(dt)
 	if err != nil {
-		cfg, err2 := ParseHCL(dt, fn)
+		f, sc, err2 := ParseHCLFile(dt, fn)
 		if err2 != nil {
-			return nil, errors.Errorf("failed to parse %s: parsing yaml: %s, parsing hcl: %s", fn, err.Error(), err2.Error())
+			return nil, nil, nil, errors.Errorf("failed to parse %s: parsing yaml: %s, parsing hcl: %s", fn, err.Error(), err2.Error())
 		}
-		return cfg, nil
+		return nil, f, sc, nil
 	}
-	return cfg, nil
+	return cfg, nil, nil, nil
 }
 
 type Config struct {

bake/hcl.go

@@ -12,9 +12,9 @@ import (
 	"github.com/hashicorp/hcl/v2/ext/tryfunc"
 	"github.com/hashicorp/hcl/v2/ext/typeexpr"
 	"github.com/hashicorp/hcl/v2/ext/userfunc"
-	"github.com/hashicorp/hcl/v2/hclsimple"
+	"github.com/hashicorp/hcl/v2/gohcl"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
-	"github.com/hashicorp/hcl/v2/json"
+	hcljson "github.com/hashicorp/hcl/v2/json"
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/zclconf/go-cty/cty"
@@ -127,59 +127,74 @@ var (
 
 // Used in the first pass of decoding instead of the Config struct to disallow
 // interpolation while parsing variable blocks.
-type staticConfig struct {
+type StaticConfig struct {
 	Variables []*Variable `hcl:"variable,block"`
 	Remain    hcl.Body    `hcl:",remain"`
 }
 
-func ParseHCL(dt []byte, fn string) (_ *Config, err error) {
-	if strings.HasSuffix(fn, ".json") || strings.HasSuffix(fn, ".hcl") {
-		return parseHCL(dt, fn)
+func mergeStaticConfig(scs []*StaticConfig) *StaticConfig {
+	if len(scs) == 0 {
+		return nil
 	}
-	cfg, err := parseHCL(dt, fn+".hcl")
-	if err != nil {
-		cfg2, err2 := parseHCL(dt, fn+".json")
-		if err2 == nil {
-			return cfg2, nil
-		}
+	sc := scs[0]
+	for _, s := range scs[1:] {
+		sc.Variables = append(sc.Variables, s.Variables...)
 	}
-	return cfg, err
+	return sc
 }
 
-func parseHCL(dt []byte, fn string) (_ *Config, err error) {
+func ParseHCLFile(dt []byte, fn string) (*hcl.File, *StaticConfig, error) {
+	if strings.HasSuffix(fn, ".json") || strings.HasSuffix(fn, ".hcl") {
+		return parseHCLFile(dt, fn)
+	}
+	f, sc, err := parseHCLFile(dt, fn+".hcl")
+	if err != nil {
+		f, sc, err2 := parseHCLFile(dt, fn+".json")
+		if err2 == nil {
+			return f, sc, nil
+		}
+	}
+	return f, sc, err
+}
+
+func parseHCLFile(dt []byte, fn string) (f *hcl.File, _ *StaticConfig, err error) {
 	defer func() {
 		err = formatHCLError(dt, err)
 	}()
 
 	// Decode user defined functions, first parsing as hcl and falling back to
 	// json, returning errors based on the file suffix.
-	file, hcldiags := hclsyntax.ParseConfig(dt, fn, hcl.Pos{Line: 1, Column: 1})
+	f, hcldiags := hclsyntax.ParseConfig(dt, fn, hcl.Pos{Line: 1, Column: 1})
 	if hcldiags.HasErrors() {
 		var jsondiags hcl.Diagnostics
-		file, jsondiags = json.Parse(dt, fn)
+		f, jsondiags = hcljson.Parse(dt, fn)
 		if jsondiags.HasErrors() {
 			fnl := strings.ToLower(fn)
 			if strings.HasSuffix(fnl, ".json") {
-				return nil, jsondiags
+				return nil, nil, jsondiags
 			}
-			return nil, hcldiags
+			return nil, nil, hcldiags
 		}
 	}
 
-	var sc staticConfig
-
+	var sc StaticConfig
 	// Decode only variable blocks without interpolation.
-	if err := hclsimple.Decode(fn, dt, nil, &sc); err != nil {
-		return nil, err
+	if err := gohcl.DecodeBody(f.Body, nil, &sc); err != nil {
+		return nil, nil, err
 	}
 
+	return f, &sc, nil
+}
+
+func ParseHCL(b hcl.Body, sc *StaticConfig) (_ *Config, err error) {
+
 	// Set all variables to their default value if defined.
 	variables := make(map[string]cty.Value)
 	for _, variable := range sc.Variables {
 		variables[variable.Name] = cty.StringVal(variable.Default)
 	}
 
-	userFunctions, _, diags := userfunc.DecodeUserFunctions(file.Body, "function", func() *hcl.EvalContext {
+	userFunctions, _, diags := userfunc.DecodeUserFunctions(b, "function", func() *hcl.EvalContext {
 		return &hcl.EvalContext{
 			Functions: stdlibFunctions,
 			Variables: variables,
@@ -214,7 +229,7 @@ func parseHCL(dt []byte, fn string) (_ *Config, err error) {
 	var c Config
 
 	// Decode with variables and functions.
-	if err := hclsimple.Decode(fn, dt, ctx, &c); err != nil {
+	if err := gohcl.DecodeBody(b, ctx, &c); err != nil {
 		return nil, err
 	}
 	return &c, nil

bake/hcl_test.go

@@ -44,7 +44,7 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.hcl")
+		c, err := ParseFile(dt, "docker-bake.hcl")
 		require.NoError(t, err)
 
 		require.Equal(t, 1, len(c.Groups))
@@ -104,7 +104,7 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.json")
+		c, err := ParseFile(dt, "docker-bake.json")
 		require.NoError(t, err)
 
 		require.Equal(t, 1, len(c.Groups))
@@ -141,7 +141,7 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.hcl")
+		c, err := ParseFile(dt, "docker-bake.hcl")
 		require.NoError(t, err)
 
 		require.Equal(t, 1, len(c.Groups))
@@ -171,7 +171,7 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.hcl")
+		c, err := ParseFile(dt, "docker-bake.hcl")
 		require.NoError(t, err)
 
 		require.Equal(t, 1, len(c.Groups))
@@ -200,7 +200,7 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.hcl")
+		c, err := ParseFile(dt, "docker-bake.hcl")
 		require.NoError(t, err)
 
 		require.Equal(t, 1, len(c.Groups))
@@ -213,7 +213,7 @@ func TestParseHCL(t *testing.T) {
 
 		os.Setenv("BUILD_NUMBER", "456")
 
-		c, err = ParseHCL(dt, "docker-bake.hcl")
+		c, err = ParseFile(dt, "docker-bake.hcl")
 		require.NoError(t, err)
 
 		require.Equal(t, 1, len(c.Groups))
@@ -246,7 +246,7 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		_, err := ParseHCL(dt, "docker-bake.hcl")
+		_, err := ParseFile(dt, "docker-bake.hcl")
 		require.Error(t, err)
 		require.Contains(t, err.Error(), "docker-bake.hcl:7,17-37: Variables not allowed; Variables may not be used here.")
 	})
@@ -266,7 +266,7 @@ func TestParseHCL(t *testing.T) {
 		}
 		`)
 
-		c, err := ParseHCL(dt, "docker-bake.hcl")
+		c, err := ParseFile(dt, "docker-bake.hcl")
 		require.NoError(t, err)
 
 		require.Equal(t, 1, len(c.Targets))
@@ -275,11 +275,54 @@ func TestParseHCL(t *testing.T) {
 
 		os.Setenv("REPO", "docker/buildx")
 
-		c, err = ParseHCL(dt, "docker-bake.hcl")
+		c, err = ParseFile(dt, "docker-bake.hcl")
 		require.NoError(t, err)
 
 		require.Equal(t, 1, len(c.Targets))
 		require.Equal(t, c.Targets[0].Name, "webapp")
 		require.Equal(t, []string{"docker/buildx:v1"}, c.Targets[0].Tags)
 	})
+
+	t.Run("MultiFileSharedVariables", func(t *testing.T) {
+		dt := []byte(`
+		variable "FOO" {
+			default = "abc"
+		}
+		target "app" {
+			args = {
+				v1 = "pre-${FOO}"
+			}
+		}
+		`)
+		dt2 := []byte(`
+		target "app" {
+			args = {
+				v2 = "${FOO}-post"
+			}
+		}
+		`)
+
+		c, err := parseFiles([]File{
+			{Data: dt, Name: "c1.hcl"},
+			{Data: dt2, Name: "c2.hcl"},
+		})
+		require.NoError(t, err)
+		require.Equal(t, 1, len(c.Targets))
+		require.Equal(t, c.Targets[0].Name, "app")
+		require.Equal(t, "pre-abc", c.Targets[0].Args["v1"])
+		require.Equal(t, "abc-post", c.Targets[0].Args["v2"])
+
+		os.Setenv("FOO", "def")
+
+		c, err = parseFiles([]File{
+			{Data: dt, Name: "c1.hcl"},
+			{Data: dt2, Name: "c2.hcl"},
+		})
+		require.NoError(t, err)
+
+		require.Equal(t, 1, len(c.Targets))
+		require.Equal(t, c.Targets[0].Name, "app")
+		require.Equal(t, "pre-def", c.Targets[0].Args["v1"])
+		require.Equal(t, "def-post", c.Targets[0].Args["v2"])
+	})
 }

vendor/github.com/hashicorp/hcl/v2/hclsimple/hclsimple.go

@@ -1,108 +0,0 @@
-// Package hclsimple is a higher-level entry point for loading HCL
-// configuration files directly into Go struct values in a single step.
-//
-// This package is more opinionated than the rest of the HCL API. See the
-// documentation for function Decode for more information.
-package hclsimple
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/hashicorp/hcl/v2"
-	"github.com/hashicorp/hcl/v2/gohcl"
-	"github.com/hashicorp/hcl/v2/hclsyntax"
-	"github.com/hashicorp/hcl/v2/json"
-)
-
-// Decode parses, decodes, and evaluates expressions in the given HCL source
-// code, in a single step.
-//
-// The main HCL API is built to allow applications that need to decompose
-// the processing steps into a pipeline, with different tasks done by
-// different parts of the program: parsing the source code into an abstract
-// representation, analysing the block structure, evaluating expressions,
-// and then extracting the results into a form consumable by the rest of
-// the program.
-//
-// This function does all of those steps in one call, going directly from
-// source code to a populated Go struct value.
-//
-// The "filename" and "src" arguments describe the input configuration. The
-// filename is used to add source location context to any returned error
-// messages and its suffix will choose one of the two supported syntaxes:
-// ".hcl" for native syntax, and ".json" for HCL JSON. The src must therefore
-// contain a sequence of bytes that is valid for the selected syntax.
-//
-// The "ctx" argument provides variables and functions for use during
-// expression evaluation. Applications that need no variables nor functions
-// can just pass nil.
-//
-// The "target" argument must be a pointer to a value of a struct type,
-// with struct tags as defined by the sibling package "gohcl".
-//
-// The return type is error but any non-nil error is guaranteed to be
-// type-assertable to hcl.Diagnostics for applications that wish to access
-// the full error details.
-//
-// This is a very opinionated function that is intended to serve the needs of
-// applications that are just using HCL for simple configuration and don't
-// need detailed control over the decoding process. Because this function is
-// just wrapping functionality elsewhere, if it doesn't meet your needs then
-// please consider copying it into your program and adapting it as needed.
-func Decode(filename string, src []byte, ctx *hcl.EvalContext, target interface{}) error {
-	var file *hcl.File
-	var diags hcl.Diagnostics
-
-	switch suffix := strings.ToLower(filepath.Ext(filename)); suffix {
-	case ".hcl":
-		file, diags = hclsyntax.ParseConfig(src, filename, hcl.Pos{Line: 1, Column: 1})
-	case ".json":
-		file, diags = json.Parse(src, filename)
-	default:
-		diags = diags.Append(&hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Unsupported file format",
-			Detail:   fmt.Sprintf("Cannot read from %s: unrecognized file format suffix %q.", filename, suffix),
-		})
-		return diags
-	}
-	if diags.HasErrors() {
-		return diags
-	}
-
-	diags = gohcl.DecodeBody(file.Body, ctx, target)
-	if diags.HasErrors() {
-		return diags
-	}
-	return nil
-}
-
-// DecodeFile is a wrapper around Decode that first reads the given filename
-// from disk. See the Decode documentation for more information.
-func DecodeFile(filename string, ctx *hcl.EvalContext, target interface{}) error {
-	src, err := ioutil.ReadFile(filename)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return hcl.Diagnostics{
-				{
-					Severity: hcl.DiagError,
-					Summary:  "Configuration file not found",
-					Detail:   fmt.Sprintf("The configuration file %s does not exist.", filename),
-				},
-			}
-		}
-		return hcl.Diagnostics{
-			{
-				Severity: hcl.DiagError,
-				Summary:  "Failed to read configuration",
-				Detail:   fmt.Sprintf("Can't read %s: %s.", filename, err),
-			},
-		}
-	}
-
-	return Decode(filename, src, ctx, target)
-}

vendor/modules.txt

@@ -227,7 +227,6 @@ github.com/hashicorp/hcl/v2/ext/tryfunc
 github.com/hashicorp/hcl/v2/ext/typeexpr
 github.com/hashicorp/hcl/v2/ext/userfunc
 github.com/hashicorp/hcl/v2/gohcl
-github.com/hashicorp/hcl/v2/hclsimple
 github.com/hashicorp/hcl/v2/hclsyntax
 github.com/hashicorp/hcl/v2/hclwrite
 github.com/hashicorp/hcl/v2/json