bake: initial implementation

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

bake/bake.go

@@ -0,0 +1,344 @@
+package bake
+
+import (
+	"context"
+	"io/ioutil"
+	"strings"
+
+	"github.com/moby/buildkit/session/auth/authprovider"
+	"github.com/pkg/errors"
+	"github.com/tonistiigi/buildx/build"
+)
+
+func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[string]Target, error) {
+	var c Config
+	for _, f := range files {
+		cfg, err := ParseFile(f)
+		if err != nil {
+			return nil, err
+		}
+		c = mergeConfig(c, *cfg)
+	}
+	if err := c.setOverrides(overrides); err != nil {
+		return nil, err
+	}
+	m := map[string]Target{}
+	for _, n := range targets {
+		for _, n := range c.ResolveGroup(n) {
+			t, err := c.ResolveTarget(n)
+			if err != nil {
+				return nil, err
+			}
+			if t != nil {
+				m[n] = *t
+			}
+		}
+	}
+	return m, nil
+}
+
+func ParseFile(fn string) (*Config, error) {
+	dt, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return nil, err
+	}
+
+	fnl := strings.ToLower(fn)
+	if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
+		return ParseCompose(dt)
+	}
+
+	if strings.HasSuffix(fnl, ".json") || strings.HasSuffix(fnl, ".hcl") {
+		return ParseHCL(dt)
+	}
+
+	cfg, err := ParseCompose(dt)
+	if err != nil {
+		cfg, err2 := ParseHCL(dt)
+		if err2 != nil {
+			return nil, errors.Errorf("failed to parse %s: parsing yaml: %s, parsing hcl: %s", fn, err.Error(), err2.Error())
+		}
+		return cfg, nil
+	}
+	return cfg, nil
+}
+
+type Config struct {
+	Group  map[string]Group
+	Target map[string]Target
+}
+
+func mergeConfig(c1, c2 Config) Config {
+	for k, g := range c2.Group {
+		if c1.Group == nil {
+			c1.Group = map[string]Group{}
+		}
+		c1.Group[k] = g
+	}
+
+	for k, t := range c2.Target {
+		if c1.Target == nil {
+			c1.Target = map[string]Target{}
+		}
+		if base, ok := c1.Target[k]; ok {
+			t = merge(base, t)
+		}
+		c1.Target[k] = t
+	}
+	return c1
+}
+
+func (c Config) setOverrides(v []string) error {
+	for _, v := range v {
+		parts := strings.SplitN(v, "=", 2)
+		if len(parts) != 2 {
+			return errors.Errorf("invalid override %s, expected target.name=value", v)
+		}
+		keys := strings.SplitN(parts[0], ".", 3)
+		if len(keys) < 2 {
+			return errors.Errorf("invalid override key %s, expected target.name", parts[0])
+		}
+
+		name := keys[0]
+
+		t, ok := c.Target[name]
+		if !ok {
+			return errors.Errorf("unknown target %s", name)
+		}
+
+		switch keys[1] {
+		case "context":
+			t.Context = parts[1]
+		case "dockerfile":
+			t.Dockerfile = parts[1]
+		case "args":
+			if len(keys) != 3 {
+				return errors.Errorf("invalid key %s, args requires name", parts[0])
+			}
+			if t.Args == nil {
+				t.Args = map[string]string{}
+			}
+			t.Args[keys[2]] = parts[1]
+		case "labels":
+			if len(keys) != 3 {
+				return errors.Errorf("invalid key %s, lanels requires name", parts[0])
+			}
+			if t.Labels == nil {
+				t.Labels = map[string]string{}
+			}
+			t.Labels[keys[2]] = parts[1]
+		case "tags":
+			t.Tags = append(t.Tags, parts[1])
+		case "cache-from":
+			t.CacheFrom = append(t.CacheFrom, parts[1])
+		case "target":
+			s := parts[1]
+			t.Target = &s
+		case "secrets":
+			t.Secrets = append(t.Secrets, parts[1])
+		case "ssh":
+			t.SSH = append(t.SSH, parts[1])
+		case "platform":
+			t.Platforms = append(t.Platforms, parts[1])
+		default:
+			return errors.Errorf("unknown key: %s", keys[1])
+		}
+		c.Target[name] = t
+	}
+	return nil
+}
+
+func (c Config) ResolveGroup(name string) []string {
+	return c.group(name, map[string]struct{}{})
+}
+
+func (c Config) group(name string, visited map[string]struct{}) []string {
+	if _, ok := visited[name]; ok {
+		return nil
+	}
+	g, ok := c.Group[name]
+	if !ok {
+		return []string{name}
+	}
+	visited[name] = struct{}{}
+	targets := make([]string, 0, len(g.Targets))
+	for _, t := range g.Targets {
+		targets = append(targets, c.group(t, visited)...)
+	}
+	return targets
+}
+
+func (c Config) ResolveTarget(name string) (*Target, error) {
+	return c.target(name, map[string]struct{}{})
+}
+
+func (c Config) target(name string, visited map[string]struct{}) (*Target, error) {
+	if _, ok := visited[name]; ok {
+		return nil, nil
+	}
+	visited[name] = struct{}{}
+	t, ok := c.Target[name]
+	if !ok {
+		return nil, errors.Errorf("failed to find target %s", name)
+	}
+	var tt Target
+	for _, name := range t.Inherits {
+		t, err := c.target(name, visited)
+		if err != nil {
+			return nil, err
+		}
+		if t != nil {
+			tt = merge(tt, *t)
+		}
+	}
+	t.Inherits = nil
+	tt = merge(merge(defaultTarget(), t), tt)
+	tt.normalize()
+	return &tt, nil
+}
+
+type Group struct {
+	Targets []string
+	// Target // TODO?
+}
+
+type Target struct {
+	Inherits   []string          `json:"inherits,omitempty"`
+	Context    string            `json:"context,omitempty"`
+	Dockerfile string            `json:"dockerfile,omitempty"`
+	Args       map[string]string `json:"args,omitempty"`
+	Labels     map[string]string `json:"labels,omitempty"`
+	Tags       []string          `json:"tags,omitempty"`
+	CacheFrom  []string          `json:"cache-from,omitempty"`
+	Target     *string           `json:"target,omitempty"`
+	Secrets    []string          `json:"secret,omitempty"`
+	SSH        []string          `json:"ssh,omitempty"`
+	Platforms  []string          `json:"platform,omitempty"`
+}
+
+func (t *Target) normalize() {
+	t.Tags = removeDupes(t.Tags)
+	t.Secrets = removeDupes(t.Secrets)
+	t.SSH = removeDupes(t.SSH)
+	t.Platforms = removeDupes(t.Platforms)
+}
+
+func TargetsToBuildOpt(m map[string]Target) (map[string]build.Options, error) {
+	m2 := make(map[string]build.Options, len(m))
+	for k, v := range m {
+		bo, err := toBuildOpt(v)
+		if err != nil {
+			return nil, err
+		}
+		m2[k] = *bo
+	}
+	return m2, nil
+}
+
+func toBuildOpt(t Target) (*build.Options, error) {
+	if t.Context == "-" {
+		return nil, errors.Errorf("context from stdin not allowed in bake")
+	}
+	if t.Dockerfile == "-" {
+		return nil, errors.Errorf("dockerfile from stdin not allowed in bake")
+	}
+
+	bo := &build.Options{
+		Inputs: build.Inputs{
+			ContextPath:    t.Context,
+			DockerfilePath: t.Dockerfile,
+		},
+		Tags:      t.Tags,
+		BuildArgs: t.Args,
+		Labels:    t.Labels,
+		// CacheFrom: t.CacheFrom,
+	}
+
+	platforms, err := build.ParsePlatformSpecs(t.Platforms)
+	if err != nil {
+		return nil, err
+	}
+	bo.Platforms = platforms
+
+	bo.Session = append(bo.Session, authprovider.NewDockerAuthProvider())
+
+	secrets, err := build.ParseSecretSpecs(t.Secrets)
+	if err != nil {
+		return nil, err
+	}
+	bo.Session = append(bo.Session, secrets)
+
+	ssh, err := build.ParseSSHSpecs(t.SSH)
+	if err != nil {
+		return nil, err
+	}
+	bo.Session = append(bo.Session, ssh)
+
+	if t.Target != nil {
+		bo.Target = *t.Target
+	}
+
+	return bo, nil
+}
+
+func defaultTarget() Target {
+	return Target{
+		Context:    ".",
+		Dockerfile: "Dockerfile",
+	}
+}
+
+func merge(t1, t2 Target) Target {
+	if t2.Context != "" {
+		t1.Context = t2.Context
+	}
+	if t2.Dockerfile != "" {
+		t1.Dockerfile = t2.Dockerfile
+	}
+	for k, v := range t2.Args {
+		if t1.Args == nil {
+			t1.Args = map[string]string{}
+		}
+		t1.Args[k] = v
+	}
+	for k, v := range t2.Labels {
+		if t1.Labels == nil {
+			t1.Labels = map[string]string{}
+		}
+		t1.Labels[k] = v
+	}
+	if t2.Tags != nil { // no merge
+		t1.Tags = t2.Tags
+	}
+	if t2.CacheFrom != nil {
+		t1.CacheFrom = t2.CacheFrom
+	}
+	if t2.Target != nil {
+		t1.Target = t2.Target
+	}
+	if t2.Secrets != nil { // merge
+		t1.Secrets = append(t1.Secrets, t2.Secrets...)
+	}
+	if t2.SSH != nil { // merge
+		t1.SSH = append(t1.SSH, t2.SSH...)
+	}
+	if t2.Platforms != nil { // no merge
+		t1.Platforms = t2.Platforms
+	}
+	t1.Inherits = append(t1.Inherits, t2.Inherits...)
+	return t1
+}
+
+func removeDupes(s []string) []string {
+	i := 0
+	seen := make(map[string]struct{}, len(s))
+	for _, v := range s {
+		if _, ok := seen[v]; ok {
+			continue
+		}
+		seen[v] = struct{}{}
+		s[i] = v
+		i++
+	}
+	return s[:i]
+}

bake/compose.go

@@ -0,0 +1,68 @@
+package bake
+
+import (
+	"github.com/docker/cli/cli/compose/loader"
+	composetypes "github.com/docker/cli/cli/compose/types"
+)
+
+func parseCompose(dt []byte) (*composetypes.Config, error) {
+	parsed, err := loader.ParseYAML([]byte(dt))
+	if err != nil {
+		return nil, err
+	}
+	return loader.Load(composetypes.ConfigDetails{
+		ConfigFiles: []composetypes.ConfigFile{
+			{
+				Config: parsed,
+			},
+		},
+	})
+}
+
+func ParseCompose(dt []byte) (*Config, error) {
+	cfg, err := parseCompose(dt)
+	if err != nil {
+		return nil, err
+	}
+
+	var c Config
+	if len(cfg.Services) > 0 {
+		c.Group = map[string]Group{}
+		c.Target = map[string]Target{}
+
+		var g Group
+
+		for _, s := range cfg.Services {
+			g.Targets = append(g.Targets, s.Name)
+			t := Target{
+				Context:    s.Build.Context,
+				Dockerfile: s.Build.Dockerfile,
+				Labels:     s.Build.Labels,
+				Args:       toMap(s.Build.Args),
+				CacheFrom:  s.Build.CacheFrom,
+				// TODO: add platforms
+			}
+			if s.Build.Target != "" {
+				t.Target = &s.Build.Target
+			}
+			if s.Image != "" {
+				t.Tags = []string{s.Image}
+			}
+			c.Target[s.Name] = t
+		}
+		c.Group["default"] = g
+
+	}
+
+	return &c, nil
+}
+
+func toMap(in composetypes.MappingWithEquals) map[string]string {
+	m := map[string]string{}
+	for k, v := range in {
+		if v != nil {
+			m[k] = *v
+		}
+	}
+	return m
+}

bake/compose_test.go

@@ -0,0 +1,41 @@
+package bake
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseCompose(t *testing.T) {
+	var dt = []byte(`
+version: "3"
+
+services:
+  db:
+    build: ./db
+    command: ./entrypoint.sh
+    image: docker.io/tonistiigi/db
+  webapp:
+    build:
+      context: ./dir
+      dockerfile: Dockerfile-alternate
+      args:
+        buildno: 123
+`)
+
+	c, err := ParseCompose(dt)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Group))
+	sort.Strings(c.Group["default"].Targets)
+	require.Equal(t, []string{"db", "webapp"}, c.Group["default"].Targets)
+
+	require.Equal(t, 2, len(c.Target))
+	require.Equal(t, "./db", c.Target["db"].Context)
+
+	require.Equal(t, "./dir", c.Target["webapp"].Context)
+	require.Equal(t, "Dockerfile-alternate", c.Target["webapp"].Dockerfile)
+	require.Equal(t, 1, len(c.Target["webapp"].Args))
+	require.Equal(t, "123", c.Target["webapp"].Args["buildno"])
+}

bake/hcl.go

@@ -0,0 +1,11 @@
+package bake
+
+import "github.com/hashicorp/hcl"
+
+func ParseHCL(dt []byte) (*Config, error) {
+	var c Config
+	if err := hcl.Unmarshal(dt, &c); err != nil {
+		return nil, err
+	}
+	return &c, nil
+}

bake/hcl_test.go

@@ -0,0 +1,57 @@
+package bake
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseHCL(t *testing.T) {
+	var dt = []byte(`
+	group "default" {
+		targets = ["db", "webapp"]
+	}
+
+	target "db" {
+		context = "./db"
+		tags = ["docker.io/tonistiigi/db"]
+	}
+
+	target "webapp" {
+		context = "./dir"
+		dockerfile = "Dockerfile-alternate"
+		args = {
+			buildno = "123"
+		}
+	}
+
+	target "cross" {
+		platforms = [
+			"linux/amd64",
+			"linux/arm64"
+		]
+	}
+
+	target "webapp-plus" {
+		inherits = ["webapp", "cross"]
+		args = {
+			IAMCROSS = "true"
+		}
+	}
+	`)
+
+	c, err := ParseHCL(dt)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(c.Group))
+	require.Equal(t, []string{"db", "webapp"}, c.Group["default"].Targets)
+
+	require.Equal(t, 4, len(c.Target))
+	require.Equal(t, "./db", c.Target["db"].Context)
+
+	require.Equal(t, 1, len(c.Target["webapp"].Args))
+	require.Equal(t, "123", c.Target["webapp"].Args["buildno"])
+
+	require.Equal(t, 2, len(c.Target["cross"].Platforms))
+	require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Target["cross"].Platforms)
+}