controller: strongly type the controller api

Strongly typing the API allows us to perform all command line parsing
fully on the client-side, where we have access to the client local
directory and all the client environment variables, which may not be
available on the remote server.

Additionally, the controller api starts to look a lot like
build.Options, so at some point in the future there may be an
oppportunity to merge the two, which would allow both build and bake to
execute through the controller, instead of needing to maintain multiple
code paths.

Signed-off-by: Justin Chadwell <me@jedevc.com>

util/buildflags/cache.go

@@ -7,12 +7,12 @@ import (
 	"strings"
 
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
-	"github.com/moby/buildkit/client"
+	controllerapi "github.com/docker/buildx/controller/pb"
 	"github.com/pkg/errors"
 )
 
-func ParseCacheEntry(in []string) ([]client.CacheOptionsEntry, error) {
-	imports := make([]client.CacheOptionsEntry, 0, len(in))
+func ParseCacheEntry(in []string) ([]*controllerapi.CacheOptionsEntry, error) {
+	outs := make([]*controllerapi.CacheOptionsEntry, 0, len(in))
 	for _, in := range in {
 		csvReader := csv.NewReader(strings.NewReader(in))
 		fields, err := csvReader.Read()
@@ -21,14 +21,15 @@ func ParseCacheEntry(in []string) ([]client.CacheOptionsEntry, error) {
 		}
 		if isRefOnlyFormat(fields) {
 			for _, field := range fields {
-				imports = append(imports, client.CacheOptionsEntry{
+				outs = append(outs, &controllerapi.CacheOptionsEntry{
 					Type:  "registry",
 					Attrs: map[string]string{"ref": field},
 				})
 			}
 			continue
 		}
-		im := client.CacheOptionsEntry{
+
+		out := controllerapi.CacheOptionsEntry{
 			Attrs: map[string]string{},
 		}
 		for _, field := range fields {
@@ -40,21 +41,21 @@ func ParseCacheEntry(in []string) ([]client.CacheOptionsEntry, error) {
 			value := parts[1]
 			switch key {
 			case "type":
-				im.Type = value
+				out.Type = value
 			default:
-				im.Attrs[key] = value
+				out.Attrs[key] = value
 			}
 		}
-		if im.Type == "" {
+		if out.Type == "" {
 			return nil, errors.Errorf("type required form> %q", in)
 		}
-		if !addGithubToken(&im) {
+		if !addGithubToken(&out) {
 			continue
 		}
-		addAwsCredentials(&im)
-		imports = append(imports, im)
+		addAwsCredentials(&out)
+		outs = append(outs, &out)
 	}
-	return imports, nil
+	return outs, nil
 }
 
 func isRefOnlyFormat(in []string) bool {
@@ -66,7 +67,7 @@ func isRefOnlyFormat(in []string) bool {
 	return true
 }
 
-func addGithubToken(ci *client.CacheOptionsEntry) bool {
+func addGithubToken(ci *controllerapi.CacheOptionsEntry) bool {
 	if ci.Type != "gha" {
 		return true
 	}
@@ -83,7 +84,7 @@ func addGithubToken(ci *client.CacheOptionsEntry) bool {
 	return ci.Attrs["token"] != "" && ci.Attrs["url"] != ""
 }
 
-func addAwsCredentials(ci *client.CacheOptionsEntry) {
+func addAwsCredentials(ci *controllerapi.CacheOptionsEntry) {
 	if ci.Type != "s3" {
 		return
 	}

util/buildflags/context.go

@@ -0,0 +1,28 @@
+package buildflags
+
+import (
+	"strings"
+
+	"github.com/docker/distribution/reference"
+	"github.com/pkg/errors"
+)
+
+func ParseContextNames(values []string) (map[string]string, error) {
+	if len(values) == 0 {
+		return nil, nil
+	}
+	result := make(map[string]string, len(values))
+	for _, value := range values {
+		kv := strings.SplitN(value, "=", 2)
+		if len(kv) != 2 {
+			return nil, errors.Errorf("invalid context value: %s, expected key=value", value)
+		}
+		named, err := reference.ParseNormalizedNamed(kv[0])
+		if err != nil {
+			return nil, errors.Wrapf(err, "invalid context name %s", kv[0])
+		}
+		name := strings.TrimSuffix(reference.FamiliarString(named), ":latest")
+		result[name] = kv[1]
+	}
+	return result, nil
+}

util/buildflags/export.go

@@ -0,0 +1,76 @@
+package buildflags
+
+import (
+	"encoding/csv"
+	"strings"
+
+	controllerapi "github.com/docker/buildx/controller/pb"
+	"github.com/moby/buildkit/client"
+	"github.com/pkg/errors"
+)
+
+func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) {
+	var outs []*controllerapi.ExportEntry
+	if len(inp) == 0 {
+		return nil, nil
+	}
+	for _, s := range inp {
+		csvReader := csv.NewReader(strings.NewReader(s))
+		fields, err := csvReader.Read()
+		if err != nil {
+			return nil, err
+		}
+
+		out := controllerapi.ExportEntry{
+			Attrs: map[string]string{},
+		}
+		if len(fields) == 1 && fields[0] == s && !strings.HasPrefix(s, "type=") {
+			if s != "-" {
+				outs = append(outs, &controllerapi.ExportEntry{
+					Type:        client.ExporterLocal,
+					Destination: s,
+				})
+				continue
+			}
+			out = controllerapi.ExportEntry{
+				Type:        client.ExporterTar,
+				Destination: s,
+			}
+		}
+
+		if out.Type == "" {
+			for _, field := range fields {
+				parts := strings.SplitN(field, "=", 2)
+				if len(parts) != 2 {
+					return nil, errors.Errorf("invalid value %s", field)
+				}
+				key := strings.TrimSpace(strings.ToLower(parts[0]))
+				value := parts[1]
+				switch key {
+				case "type":
+					out.Type = value
+				default:
+					out.Attrs[key] = value
+				}
+			}
+		}
+		if out.Type == "" {
+			return nil, errors.Errorf("type is required for output")
+		}
+
+		if out.Type == "registry" {
+			out.Type = client.ExporterImage
+			if _, ok := out.Attrs["push"]; !ok {
+				out.Attrs["push"] = "true"
+			}
+		}
+
+		if dest, ok := out.Attrs["dest"]; ok {
+			out.Destination = dest
+			delete(out.Attrs, "dest")
+		}
+
+		outs = append(outs, &out)
+	}
+	return outs, nil
+}

util/buildflags/output.go

@@ -1,142 +0,0 @@
-package buildflags
-
-import (
-	"encoding/csv"
-	"io"
-	"os"
-	"strconv"
-	"strings"
-
-	"github.com/containerd/console"
-	"github.com/moby/buildkit/client"
-	"github.com/pkg/errors"
-)
-
-func ParseOutputs(inp []string) ([]client.ExportEntry, error) {
-	var outs []client.ExportEntry
-	if len(inp) == 0 {
-		return nil, nil
-	}
-	for _, s := range inp {
-		csvReader := csv.NewReader(strings.NewReader(s))
-		fields, err := csvReader.Read()
-		if err != nil {
-			return nil, err
-		}
-
-		out := client.ExportEntry{
-			Attrs: map[string]string{},
-		}
-		if len(fields) == 1 && fields[0] == s && !strings.HasPrefix(s, "type=") {
-			if s != "-" {
-				outs = append(outs, client.ExportEntry{
-					Type:      client.ExporterLocal,
-					OutputDir: s,
-				})
-				continue
-			}
-			out = client.ExportEntry{
-				Type: client.ExporterTar,
-				Attrs: map[string]string{
-					"dest": s,
-				},
-			}
-		}
-
-		if out.Type == "" {
-			for _, field := range fields {
-				parts := strings.SplitN(field, "=", 2)
-				if len(parts) != 2 {
-					return nil, errors.Errorf("invalid value %s", field)
-				}
-				key := strings.TrimSpace(strings.ToLower(parts[0]))
-				value := parts[1]
-				switch key {
-				case "type":
-					out.Type = value
-				default:
-					out.Attrs[key] = value
-				}
-			}
-		}
-		if out.Type == "" {
-			return nil, errors.Errorf("type is required for output")
-		}
-
-		supportFile := false
-		supportDir := false
-		switch out.Type {
-		case client.ExporterLocal:
-			supportDir = true
-		case client.ExporterTar:
-			supportFile = true
-		case client.ExporterOCI, client.ExporterDocker:
-			tar, err := strconv.ParseBool(out.Attrs["tar"])
-			if err != nil {
-				tar = true
-			}
-			supportFile = tar
-			supportDir = !tar
-		case "registry":
-			out.Type = client.ExporterImage
-			if _, ok := out.Attrs["push"]; !ok {
-				out.Attrs["push"] = "true"
-			}
-		}
-
-		dest, ok := out.Attrs["dest"]
-		if supportDir {
-			if !ok {
-				return nil, errors.Errorf("dest is required for %s exporter", out.Type)
-			}
-			if dest == "-" {
-				return nil, errors.Errorf("dest cannot be stdout for %s exporter", out.Type)
-			}
-
-			fi, err := os.Stat(dest)
-			if err != nil && !os.IsNotExist(err) {
-				return nil, errors.Wrapf(err, "invalid destination directory: %s", dest)
-			}
-			if err == nil && !fi.IsDir() {
-				return nil, errors.Errorf("destination directory %s is a file", dest)
-			}
-			out.OutputDir = dest
-		}
-		if supportFile {
-			if !ok && out.Type != client.ExporterDocker {
-				dest = "-"
-			}
-			if dest == "-" {
-				if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
-					return nil, errors.Errorf("dest file is required for %s exporter. refusing to write to console", out.Type)
-				}
-				out.Output = wrapWriteCloser(os.Stdout)
-			} else if dest != "" {
-				fi, err := os.Stat(dest)
-				if err != nil && !os.IsNotExist(err) {
-					return nil, errors.Wrapf(err, "invalid destination file: %s", dest)
-				}
-				if err == nil && fi.IsDir() {
-					return nil, errors.Errorf("destination file %s is a directory", dest)
-				}
-				f, err := os.Create(dest)
-				if err != nil {
-					return nil, errors.Errorf("failed to open %s", err)
-				}
-				out.Output = wrapWriteCloser(f)
-			}
-		}
-		if supportFile || supportDir {
-			delete(out.Attrs, "dest")
-		}
-
-		outs = append(outs, out)
-	}
-	return outs, nil
-}
-
-func wrapWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
-	return func(map[string]string) (io.WriteCloser, error) {
-		return wc, nil
-	}
-}

util/buildflags/secrets.go

@@ -4,35 +4,30 @@ import (
 	"encoding/csv"
 	"strings"
 
-	"github.com/moby/buildkit/session"
-	"github.com/moby/buildkit/session/secrets/secretsprovider"
+	controllerapi "github.com/docker/buildx/controller/pb"
 	"github.com/pkg/errors"
 )
 
-func ParseSecretSpecs(sl []string) (session.Attachable, error) {
-	fs := make([]secretsprovider.Source, 0, len(sl))
+func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) {
+	fs := make([]*controllerapi.Secret, 0, len(sl))
 	for _, v := range sl {
 		s, err := parseSecret(v)
 		if err != nil {
 			return nil, err
 		}
-		fs = append(fs, *s)
+		fs = append(fs, s)
 	}
-	store, err := secretsprovider.NewStore(fs)
-	if err != nil {
-		return nil, err
-	}
-	return secretsprovider.NewSecretProvider(store), nil
+	return fs, nil
 }
 
-func parseSecret(value string) (*secretsprovider.Source, error) {
+func parseSecret(value string) (*controllerapi.Secret, error) {
 	csvReader := csv.NewReader(strings.NewReader(value))
 	fields, err := csvReader.Read()
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse csv secret")
 	}
 
-	fs := secretsprovider.Source{}
+	fs := controllerapi.Secret{}
 
 	var typ string
 	for _, field := range fields {

util/buildflags/ssh.go

@@ -3,32 +3,27 @@ package buildflags
 import (
 	"strings"
 
-	"github.com/moby/buildkit/session"
-	"github.com/moby/buildkit/session/sshforward/sshprovider"
+	controllerapi "github.com/docker/buildx/controller/pb"
 	"github.com/moby/buildkit/util/gitutil"
 )
 
-func ParseSSHSpecs(sl []string) (session.Attachable, error) {
-	configs := make([]sshprovider.AgentConfig, 0, len(sl))
-	for _, v := range sl {
-		c, err := parseSSH(v)
-		if err != nil {
-			return nil, err
-		}
-		configs = append(configs, *c)
+func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) {
+	var outs []*controllerapi.SSH
+	if len(sl) == 0 {
+		return nil, nil
 	}
-	return sshprovider.NewSSHAgentProvider(configs)
-}
 
-func parseSSH(value string) (*sshprovider.AgentConfig, error) {
-	parts := strings.SplitN(value, "=", 2)
-	cfg := sshprovider.AgentConfig{
-		ID: parts[0],
+	for _, s := range sl {
+		parts := strings.SplitN(s, "=", 2)
+		out := controllerapi.SSH{
+			ID: parts[0],
+		}
+		if len(parts) > 1 {
+			out.Paths = strings.Split(parts[1], ",")
+		}
+		outs = append(outs, &out)
 	}
-	if len(parts) > 1 {
-		cfg.Paths = strings.Split(parts[1], ",")
-	}
-	return &cfg, nil
+	return outs, nil
 }
 
 // IsGitSSH returns true if the given repo URL is accessed over ssh