driver: allow setting buildkit config file

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
Co-Authored-By: Tibor Vass <tiborvass@users.noreply.github.com>

commands/create.go

@@ -24,6 +24,7 @@ type createOptions struct {
 	actionLeave  bool
 	use          bool
 	flags        string
+	configFile   string
 	// upgrade      bool // perform upgrade of the driver
 }
 
@@ -138,7 +139,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 				return err
 			}
 		}
-		if err := ng.Update(in.nodeName, ep, in.platform, len(args) > 0, in.actionAppend, flags); err != nil {
+		if err := ng.Update(in.nodeName, ep, in.platform, len(args) > 0, in.actionAppend, flags, in.configFile); err != nil {
 			return err
 		}
 	}
@@ -184,6 +185,7 @@ func createCmd(dockerCli command.Cli) *cobra.Command {
 	flags.StringVar(&options.driver, "driver", "", fmt.Sprintf("Driver to use (available: %v)", drivers))
 	flags.StringVar(&options.nodeName, "node", "", "Create/modify node with given name")
 	flags.StringVar(&options.flags, "buildkitd-flags", "", "Flags for buildkitd daemon")
+	flags.StringVar(&options.configFile, "config", "", "BuildKit config file")
 	flags.StringArrayVar(&options.platform, "platform", []string{}, "Fixed platforms for current node")
 
 	flags.BoolVar(&options.actionAppend, "append", false, "Append a node to builder instead of changing it")

commands/util.go

@@ -174,7 +174,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
 				// TODO: replace the following line with dockerclient.WithAPIVersionNegotiation option in clientForEndpoint
 				dockerapi.NegotiateAPIVersion(ctx)
 
-				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, n.Flags)
+				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, n.Flags, n.ConfigFile)
 				if err != nil {
 					di.Err = err
 					return nil
@@ -251,7 +251,7 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli) ([]build.Driv
 		return driversForNodeGroup(ctx, dockerCli, ng)
 	}
 
-	d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), nil)
+	d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), nil, "")
 	if err != nil {
 		return nil, err
 	}

driver/docker-container/driver.go

@@ -1,6 +1,8 @@
 package docker
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"io"
 	"io/ioutil"
@@ -74,6 +76,15 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 		if err != nil {
 			return err
 		}
+		if f := d.InitConfig.ConfigFile; f != "" {
+			buf, err := readFileToTar(f)
+			if err != nil {
+				return err
+			}
+			if err := d.DockerAPI.CopyToContainer(ctx, d.Name, "/", buf, dockertypes.CopyToContainerOptions{}); err != nil {
+				return err
+			}
+		}
 		if err := d.start(ctx, l); err != nil {
 			return err
 		}
@@ -245,3 +256,26 @@ type demux struct {
 func (d *demux) Read(dt []byte) (int, error) {
 	return d.Reader.Read(dt)
 }
+
+func readFileToTar(fn string) (*bytes.Buffer, error) {
+	buf := bytes.NewBuffer(nil)
+	tw := tar.NewWriter(buf)
+	dt, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{
+		Name: "/etc/buildkit/buildkitd.toml",
+		Size: int64(len(dt)),
+		Mode: 0644,
+	}); err != nil {
+		return nil, err
+	}
+	if _, err := tw.Write(dt); err != nil {
+		return nil, err
+	}
+	if err := tw.Close(); err != nil {
+		return nil, err
+	}
+	return buf, nil
+}

driver/docker/factory.go

@@ -44,6 +44,9 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 	if cfg.DockerAPI == nil {
 		return nil, errors.Errorf("docker driver requires docker API access")
 	}
+	if cfg.ConfigFile != "" {
+		return nil, errors.Errorf("setting config file is not supported for docker driver, use dockerd configuration file")
+	}
 
 	return &Driver{factory: f, InitConfig: cfg}, nil
 }

driver/manager.go

@@ -26,6 +26,7 @@ type InitConfig struct {
 	Name          string
 	DockerAPI     dockerclient.APIClient
 	BuildkitFlags []string
+	ConfigFile    string
 	Meta          map[string]interface{}
 }
 
@@ -71,11 +72,12 @@ func GetFactory(name string, instanceRequired bool) Factory {
 	return nil
 }
 
-func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, flags []string) (Driver, error) {
+func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, flags []string, config string) (Driver, error) {
 	ic := InitConfig{
 		DockerAPI:     api,
 		Name:          name,
 		BuildkitFlags: flags,
+		ConfigFile:    config,
 	}
 	if f == nil {
 		var err error

store/nodegroup.go

@@ -16,10 +16,11 @@ type NodeGroup struct {
 }
 
 type Node struct {
-	Name      string
+	Name       string
-	Endpoint  string
+	Endpoint   string
-	Platforms []specs.Platform
+	Platforms  []specs.Platform
-	Flags     []string
+	Flags      []string
+	ConfigFile string
 }
 
 func (ng *NodeGroup) Leave(name string) error {
@@ -34,7 +35,7 @@ func (ng *NodeGroup) Leave(name string) error {
 	return nil
 }
 
-func (ng *NodeGroup) Update(name, endpoint string, platforms []string, endpointsSet bool, actionAppend bool, flags []string) error {
+func (ng *NodeGroup) Update(name, endpoint string, platforms []string, endpointsSet bool, actionAppend bool, flags []string, configFile string) error {
 	i := ng.findNode(name)
 	if i == -1 && !actionAppend {
 		if len(ng.Nodes) > 0 {
@@ -76,10 +77,11 @@ func (ng *NodeGroup) Update(name, endpoint string, platforms []string, endpoints
 	}
 
 	n := Node{
-		Name:      name,
+		Name:       name,
-		Endpoint:  endpoint,
+		Endpoint:   endpoint,
-		Platforms: pp,
+		Platforms:  pp,
-		Flags:     flags,
+		ConfigFile: configFile,
+		Flags:      flags,
 	}
 	ng.Nodes = append(ng.Nodes, n)
 

store/nodegroup_test.go

@@ -11,16 +11,16 @@ func TestNodeGroupUpdate(t *testing.T) {
 	t.Parallel()
 
 	ng := &NodeGroup{}
-	err := ng.Update("foo", "foo0", []string{"linux/amd64"}, true, false, []string{"--debug"})
+	err := ng.Update("foo", "foo0", []string{"linux/amd64"}, true, false, []string{"--debug"}, "")
 	require.NoError(t, err)
 
-	err = ng.Update("foo1", "foo1", []string{"linux/arm64", "linux/arm/v7"}, true, true, nil)
+	err = ng.Update("foo1", "foo1", []string{"linux/arm64", "linux/arm/v7"}, true, true, nil, "")
 	require.NoError(t, err)
 
 	require.Equal(t, len(ng.Nodes), 2)
 
 	// update
-	err = ng.Update("foo", "foo2", []string{"linux/amd64", "linux/arm"}, true, false, nil)
+	err = ng.Update("foo", "foo2", []string{"linux/amd64", "linux/arm"}, true, false, nil, "")
 	require.NoError(t, err)
 
 	require.Equal(t, len(ng.Nodes), 2)
@@ -32,7 +32,7 @@ func TestNodeGroupUpdate(t *testing.T) {
 	require.Equal(t, []string(nil), ng.Nodes[1].Flags)
 
 	// duplicate endpoint
-	err = ng.Update("foo1", "foo2", nil, true, false, nil)
+	err = ng.Update("foo1", "foo2", nil, true, false, nil, "")
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "duplicate endpoint")